vriska/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge branch 'update-nix-unstable' of https://github.com/Ma27/hydra

Browse source
This commit is contained in:
Eelco Dolstra 2020-10-18 20:40:57 +02:00
parent 46a8d6d531 9cc76f6d69
commit bea7c2581f
7 changed files with 70 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
flake.lock
View file

@ -3,16 +3,15 @@
"lowdown-src": { "lowdown-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1598296217, "lastModified": 1598695561,
"narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=", "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
"owner": "edolstra", "owner": "kristapsdz",
"repo": "lowdown", "repo": "lowdown",
"rev": "c7a4e715af1e233080842db82d15b261cb74cb28", "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "edolstra", "owner": "kristapsdz",
"ref": "no-structs-in-anonymous-unions",
"repo": "lowdown", "repo": "lowdown",
"type": "github" "type": "github"
} }
@ -23,11 +22,11 @@
"nixpkgs": "nixpkgs" "nixpkgs": "nixpkgs"
}, },
"locked": { "locked": {
"lastModified": 1600161919, "lastModified": 1601119984,
"narHash": "sha256-kfxiAdd/CK09M5i2WeBbTIxDenAdlqDLkhsrMZy8qQo=", "narHash": "sha256-OBwnd2GzAXb0VtOuovgj3mv/Ax8Jyqelwh6j/TeMD5U=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nix", "repo": "nix",
"rev": "2a8017092025a7c108ca1a829a8b4f0bcf7ee2c0", "rev": "8b4a542d1767e0df7b3c0902b766f34352cb0958",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -52,11 +51,11 @@
}, },
"nixpkgs_2": { "nixpkgs_2": {
"locked": { "locked": {
"lastModified": 1599903709, "lastModified": 1600952868,
"narHash": "sha256-tinVKY7zRdSXmVOp7BAulNtIb2iP5nvNp5UAxihKXNI=", "narHash": "sha256-U1HfLLrV7gs2u0hU1YjU4aDGbTC1uYtxyNu2TWXKXMY=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "252bfe0107587d40092057f338e9ffcf7bbd90cb", "rev": "5659cb448e9b615d642c5fe52779c2223e72f7eb",
"type": "github" "type": "github"
}, },
"original": { "original": {

1
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -1,5 +1,6 @@
#include <map> #include <map>
#include <iostream> #include <iostream>
#include <thread>
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"

12
src/hydra-queue-runner/build-remote.cc
View file

@ -263,9 +263,9 @@ void State::buildRemote(ref<Store> destStore,
auto drv2 = localStore->readDerivation(input.first); auto drv2 = localStore->readDerivation(input.first);
for (auto & name : input.second) { for (auto & name : input.second) {
if (auto i = get(drv2.outputs, name)) { if (auto i = get(drv2.outputs, name)) {
auto outPath = i->path(*localStore, drv2.name); auto outPath = i->path(*localStore, drv2.name, name);
inputs.insert(outPath); inputs.insert(*outPath);
basicDrv.inputSrcs.insert(outPath); basicDrv.inputSrcs.insert(*outPath);
} }
} }
} }
@ -434,7 +434,11 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
auto outputs = step->drv->outputPaths(*localStore); StorePathSet outputs;
for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second)
outputs.insert(*i.second.second);
}
/* Get info about each output path. */ /* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos; std::map<StorePath, ValidPathInfo> infos;

19
src/hydra-queue-runner/build-result.cc
View file

@ -16,10 +16,13 @@ BuildOutput getBuildOutput(
BuildOutput res; BuildOutput res;
/* Compute the closure size. */ /* Compute the closure size. */
auto outputs = drv.outputPaths(*store); StorePathSet outputs;
StorePathSet closure; StorePathSet closure;
for (auto & output : outputs) for (auto & i : drv.outputsAndOptPaths(*store))
store->computeFSClosure(output, closure); if (i.second.second) {
store->computeFSClosure(*i.second.second, closure);
outputs.insert(*i.second.second);
}
for (auto & path : closure) { for (auto & path : closure) {
auto info = store->queryPathInfo(path); auto info = store->queryPathInfo(path);
res.closureSize += info->narSize; res.closureSize += info->narSize;
@ -104,13 +107,13 @@ BuildOutput getBuildOutput(
/* If no build products were explicitly declared, then add all /* If no build products were explicitly declared, then add all
outputs as a product of type "nix-build". */ outputs as a product of type "nix-build". */
if (!explicitProducts) { if (!explicitProducts) {
for (auto & output : drv.outputs) { for (auto & [name, output] : drv.outputs) {
BuildProduct product; BuildProduct product;
auto outPath = output.second.path(*store, drv.name); auto outPath = output.path(*store, drv.name, name);
product.path = store->printStorePath(outPath); product.path = store->printStorePath(*outPath);
product.type = "nix-build"; product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first; product.subtype = name == "out" ? "" : name;
product.name = outPath.name(); product.name = outPath->name();
auto file = narMembers.find(product.path); auto file = narMembers.find(product.path);
assert(file != narMembers.end()); assert(file != narMembers.end());

11
src/hydra-queue-runner/builder.cc
View file

@ -274,8 +274,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr); assert(stepNr);
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
addRoot(path); if (i.second.second)
addRoot(*i.second.second);
}
/* Register success in the database for all Build objects that /* Register success in the database for all Build objects that
have this step as the top-level step. Since the queue have this step as the top-level step. Since the queue
@ -463,8 +465,9 @@ void State::failStep(
/* Remember failed paths in the database so that they /* Remember failed paths in the database so that they
won't be built again. */ won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache) if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore))
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path)); if (i.second.second)
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(*i.second.second));
txn.commit(); txn.commit();
} }

9
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -256,10 +256,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
if (r.affected_rows() == 0) goto restart; if (r.affected_rows() == 0) goto restart;
for (auto & output : step->drv->outputs) for (auto & [name, output] : step->drv->outputs)
txn.exec_params0 txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name))); buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
if (status == bsBusy) if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -440,8 +440,9 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn) bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore))
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty()) if (i.second.second)
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(*i.second.second)).empty())
return true; return true;
return false; return false;
} }

40
src/hydra-queue-runner/queue-monitor.cc
View file

@ -176,16 +176,18 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>(); if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
if (!propagatedFrom) { if (!propagatedFrom) {
for (auto & output : ex.step->drv->outputPaths(*localStore)) { for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second) {
auto res = txn.exec_params auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(output)); localStore->printStorePath(*i.second.second));
if (!res[0][0].is_null()) { if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>(); propagatedFrom = res[0][0].as<BuildID>();
break; break;
} }
} }
} }
}
createBuildStep(txn, 0, build->id, ex.step, "", bsCachedFailure, "", propagatedFrom); createBuildStep(txn, 0, build->id, ex.step, "", bsCachedFailure, "", propagatedFrom);
txn.exec_params txn.exec_params
@ -221,8 +223,9 @@ bool State::getQueuedBuilds(Connection & conn,
auto drv = localStore->readDerivation(build->drvPath); auto drv = localStore->readDerivation(build->drvPath);
BuildOutput res = getBuildOutputCached(conn, destStore, drv); BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths(*localStore)) for (auto & i : drv.outputsAndOptPaths(*localStore))
addRoot(path); if (i.second.second)
addRoot(*i.second.second);
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
@ -453,10 +456,9 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */ /* Are all outputs valid? */
bool valid = true; bool valid = true;
auto outputs = step->drv->outputPaths(*localStore);
DerivationOutputs missing; DerivationOutputs missing;
for (auto & i : step->drv->outputs) for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) { if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
valid = false; valid = false;
missing.insert_or_assign(i.first, i.second); missing.insert_or_assign(i.first, i.second);
} }
@ -467,12 +469,12 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0; size_t avail = 0;
for (auto & i : missing) { for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name); auto path = i.second.path(*localStore, step->drv->name, i.first);
if (/* localStore != destStore && */ localStore->isValidPath(path)) if (/* localStore != destStore && */ localStore->isValidPath(*path))
avail++; avail++;
else if (useSubstitutes) { else if (useSubstitutes) {
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({{path, {}}}, infos); localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
if (infos.size() == 1) if (infos.size() == 1)
avail++; avail++;
} }
@ -481,37 +483,37 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) { if (missing.size() == avail) {
valid = true; valid = true;
for (auto & i : missing) { for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name); auto path = i.second.path(*localStore, step->drv->name, i.first);
try { try {
time_t startTime = time(0); time_t startTime = time(0);
if (localStore->isValidPath(path)) if (localStore->isValidPath(*path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store", printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
else { else {
printInfo("substituting output ‘%1%’ of ‘%2%’", printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
localStore->ensurePath(path); localStore->ensurePath(*path);
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
copyClosure(ref<Store>(localStore), destStore, {path}); copyClosure(ref<Store>(localStore), destStore, {*path});
time_t stopTime = time(0); time_t stopTime = time(0);
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(conn); pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
txn.commit(); txn.commit();
} }
} catch (Error & e) { } catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s", printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath), localStore->printStorePath(drvPath),
e.what()); e.what());
valid = false; valid = false;
@ -617,12 +619,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & output : drv.outputPaths(*localStore)) { for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
auto r = txn.exec_params auto r = txn.exec_params
("select id, buildStatus, releaseName, closureSize, size from Builds b " ("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build " "join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
localStore->printStorePath(output)); localStore->printStorePath(*output.second));
if (r.empty()) continue; if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>(); BuildID id = r[0][0].as<BuildID>();