ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge branch 'update-nix-unstable' of https://github.com/Ma27/hydra

Browse source
This commit is contained in:
Eelco Dolstra 2020-10-18 20:40:57 +02:00
parent 46a8d6d531 9cc76f6d69
commit bea7c2581f
7 changed files with 70 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
flake.lock
View file

@ -3,16 +3,15 @@
"lowdown-src": {
"flake": false,
"locked": {
"lastModified": 1598296217,
"narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=",
"owner": "edolstra",
"lastModified": 1598695561,
"narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
"owner": "kristapsdz",
"repo": "lowdown",
"rev": "c7a4e715af1e233080842db82d15b261cb74cb28",
"rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
"type": "github"
},
"original": {
"owner": "edolstra",
"ref": "no-structs-in-anonymous-unions",
"owner": "kristapsdz",
"repo": "lowdown",
"type": "github"
}
@ -23,11 +22,11 @@
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1600161919,
"narHash": "sha256-kfxiAdd/CK09M5i2WeBbTIxDenAdlqDLkhsrMZy8qQo=",
"lastModified": 1601119984,
"narHash": "sha256-OBwnd2GzAXb0VtOuovgj3mv/Ax8Jyqelwh6j/TeMD5U=",
"owner": "NixOS",
"repo": "nix",
"rev": "2a8017092025a7c108ca1a829a8b4f0bcf7ee2c0",
"rev": "8b4a542d1767e0df7b3c0902b766f34352cb0958",
"type": "github"
},
"original": {
@ -52,11 +51,11 @@
},
"nixpkgs_2": {
"locked": {
"lastModified": 1599903709,
"narHash": "sha256-tinVKY7zRdSXmVOp7BAulNtIb2iP5nvNp5UAxihKXNI=",
"lastModified": 1600952868,
"narHash": "sha256-U1HfLLrV7gs2u0hU1YjU4aDGbTC1uYtxyNu2TWXKXMY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "252bfe0107587d40092057f338e9ffcf7bbd90cb",
"rev": "5659cb448e9b615d642c5fe52779c2223e72f7eb",
"type": "github"
},
"original": {

1
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -1,5 +1,6 @@
#include <map>
#include <iostream>
#include <thread>
#include "shared.hh"
#include "store-api.hh"

12
src/hydra-queue-runner/build-remote.cc
View file

@ -263,9 +263,9 @@ void State::buildRemote(ref<Store> destStore,
auto drv2 = localStore->readDerivation(input.first);
for (auto & name : input.second) {
if (auto i = get(drv2.outputs, name)) {
auto outPath = i->path(*localStore, drv2.name);
inputs.insert(outPath);
basicDrv.inputSrcs.insert(outPath);
auto outPath = i->path(*localStore, drv2.name, name);
inputs.insert(*outPath);
basicDrv.inputSrcs.insert(*outPath);
}
}
}
@ -434,7 +434,11 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now();
auto outputs = step->drv->outputPaths(*localStore);
StorePathSet outputs;
for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second)
outputs.insert(*i.second.second);
}
/* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos;

19
src/hydra-queue-runner/build-result.cc
View file

@ -16,10 +16,13 @@ BuildOutput getBuildOutput(
BuildOutput res;
/* Compute the closure size. */
auto outputs = drv.outputPaths(*store);
StorePathSet outputs;
StorePathSet closure;
for (auto & output : outputs)
store->computeFSClosure(output, closure);
for (auto & i : drv.outputsAndOptPaths(*store))
if (i.second.second) {
store->computeFSClosure(*i.second.second, closure);
outputs.insert(*i.second.second);
}
for (auto & path : closure) {
auto info = store->queryPathInfo(path);
res.closureSize += info->narSize;
@ -104,13 +107,13 @@ BuildOutput getBuildOutput(
/* If no build products were explicitly declared, then add all
outputs as a product of type "nix-build". */
if (!explicitProducts) {
for (auto & output : drv.outputs) {
for (auto & [name, output] : drv.outputs) {
BuildProduct product;
auto outPath = output.second.path(*store, drv.name);
product.path = store->printStorePath(outPath);
auto outPath = output.path(*store, drv.name, name);
product.path = store->printStorePath(*outPath);
product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first;
product.name = outPath.name();
product.subtype = name == "out" ? "" : name;
product.name = outPath->name();
auto file = narMembers.find(product.path);
assert(file != narMembers.end());

11
src/hydra-queue-runner/builder.cc
View file

@ -274,8 +274,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr);
for (auto & path : step->drv->outputPaths(*localStore))
addRoot(path);
for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second)
addRoot(*i.second.second);
}
/* Register success in the database for all Build objects that
have this step as the top-level step. Since the queue
@ -463,8 +465,9 @@ void State::failStep(
/* Remember failed paths in the database so that they
won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv->outputPaths(*localStore))
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path));
for (auto & i : step->drv->outputsAndOptPaths(*localStore))
if (i.second.second)
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(*i.second.second));
txn.commit();
}

11
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -256,10 +256,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
if (r.affected_rows() == 0) goto restart;
for (auto & output : step->drv->outputs)
for (auto & [name, output] : step->drv->outputs)
txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name)));
buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -440,9 +440,10 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{
pqxx::work txn(conn);
for (auto & path : step->drv->outputPaths(*localStore))
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty())
return true;
for (auto & i : step->drv->outputsAndOptPaths(*localStore))
if (i.second.second)
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(*i.second.second)).empty())
return true;
return false;
}

50
src/hydra-queue-runner/queue-monitor.cc
View file

@ -176,13 +176,15 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
if (!propagatedFrom) {
for (auto & output : ex.step->drv->outputPaths(*localStore)) {
auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(output));
if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>();
break;
for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second) {
auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(*i.second.second));
if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>();
break;
}
}
}
}
@ -221,8 +223,9 @@ bool State::getQueuedBuilds(Connection & conn,
auto drv = localStore->readDerivation(build->drvPath);
BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths(*localStore))
addRoot(path);
for (auto & i : drv.outputsAndOptPaths(*localStore))
if (i.second.second)
addRoot(*i.second.second);
{
auto mc = startDbUpdate();
@ -453,10 +456,9 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */
bool valid = true;
auto outputs = step->drv->outputPaths(*localStore);
DerivationOutputs missing;
for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) {
if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
valid = false;
missing.insert_or_assign(i.first, i.second);
}
@ -467,12 +469,12 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0;
for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name);
if (/* localStore != destStore && */ localStore->isValidPath(path))
auto path = i.second.path(*localStore, step->drv->name, i.first);
if (/* localStore != destStore && */ localStore->isValidPath(*path))
avail++;
else if (useSubstitutes) {
SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({{path, {}}}, infos);
localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
if (infos.size() == 1)
avail++;
}
@ -481,37 +483,37 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) {
valid = true;
for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name);
auto path = i.second.path(*localStore, step->drv->name, i.first);
try {
time_t startTime = time(0);
if (localStore->isValidPath(path))
if (localStore->isValidPath(*path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(path),
localStore->printStorePath(*path),
localStore->printStorePath(drvPath));
else {
printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(path),
localStore->printStorePath(*path),
localStore->printStorePath(drvPath));
localStore->ensurePath(path);
localStore->ensurePath(*path);
// FIXME: should copy directly from substituter to destStore.
}
copyClosure(ref<Store>(localStore), destStore, {path});
copyClosure(ref<Store>(localStore), destStore, {*path});
time_t stopTime = time(0);
{
auto mc = startDbUpdate();
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
txn.commit();
}
} catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(path),
localStore->printStorePath(*path),
localStore->printStorePath(drvPath),
e.what());
valid = false;
@ -617,12 +619,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
{
pqxx::work txn(conn);
for (auto & output : drv.outputPaths(*localStore)) {
for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
auto r = txn.exec_params
("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
localStore->printStorePath(output));
localStore->printStorePath(*output.second));
if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>();