janik/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Build against nix-master

Browse source
This commit is contained in:
Eelco Dolstra 2019-12-30 22:49:26 +01:00
parent 5cac40a438
commit e7f2139e25
14 changed files with 172 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
flake.lock
View file

@ -4,20 +4,20 @@
"inputs": { "inputs": {
"nixpkgs": { "nixpkgs": {
"inputs": {}, "inputs": {},
"narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=",
"originalUrl": "nixpkgs/release-19.09", "originalUrl": "nixpkgs/release-19.09",
"url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd"
} }
}, },
"narHash": "sha256-zfFjX4SmTKGponLg7X1Lc4XHOUNEDz3czSjGaZTHXOk=", "narHash": "sha256-wiOmdFFMhuBGEcAURvj7c1LAcVbO82QjcFr+5WjuNzM=",
"originalUrl": "nix", "originalUrl": "nix",
"url": "github:NixOS/nix/2c1e05ae9389742dac637a6f051f718397eff2db" "url": "github:NixOS/nix/c7866733d7ce2836fbb43de90dd64d17b0d20753"
}, },
"nixpkgs": { "nixpkgs": {
"inputs": {}, "inputs": {},
"narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=",
"originalUrl": "nixpkgs/release-19.09", "originalUrl": "nixpkgs/release-19.09",
"url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd"
} }
}, },
"version": 3 "version": 3

2
src/hydra-eval-jobs/Makefile.am
View file

@ -1,5 +1,5 @@
bin_PROGRAMS = hydra-eval-jobs bin_PROGRAMS = hydra-eval-jobs
hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
hydra_eval_jobs_LDADD = $(NIX_LIBS) hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust
hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra

5
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -115,8 +115,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top,
done. */ done. */
auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>(); auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();
if (gcRootsDir != "" && localStore) { if (gcRootsDir != "" && localStore) {
Path root = gcRootsDir + "/" + baseNameOf(drvPath); Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
if (!pathExists(root)) localStore->addPermRoot(drvPath, root, false); if (!pathExists(root))
localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false);
} }
auto res2 = res.object("outputs"); auto res2 = res.object("outputs");

2
src/hydra-queue-runner/Makefile.am
View file

@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \ builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh token-server.hh state.hh db.hh build-result.hh counter.hh token-server.hh state.hh db.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

76
src/hydra-queue-runner/build-remote.cc
View file

@ -82,10 +82,10 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore, static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
FdSource & from, FdSink & to, const PathSet & paths, FdSource & from, FdSink & to, const StorePathSet & paths,
bool useSubstitutes = false) bool useSubstitutes = false)
{ {
PathSet closure; StorePathSet closure;
for (auto & path : paths) for (auto & path : paths)
destStore->computeFSClosure(path, closure); destStore->computeFSClosure(path, closure);
@ -94,20 +94,21 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
garbage-collect paths that are already there. Optionally, ask garbage-collect paths that are already there. Optionally, ask
the remote host to substitute missing paths. */ the remote host to substitute missing paths. */
// FIXME: substitute output pollutes our build log // FIXME: substitute output pollutes our build log
to << cmdQueryValidPaths << 1 << useSubstitutes << closure; to << cmdQueryValidPaths << 1 << useSubstitutes;
writeStorePaths(*destStore, to, closure);
to.flush(); to.flush();
/* Get back the set of paths that are already valid on the remote /* Get back the set of paths that are already valid on the remote
host. */ host. */
auto present = readStorePaths<PathSet>(*destStore, from); auto present = readStorePaths<StorePathSet>(*destStore, from);
if (present.size() == closure.size()) return; if (present.size() == closure.size()) return;
Paths sorted = destStore->topoSortPaths(closure); auto sorted = destStore->topoSortPaths(closure);
Paths missing; StorePathSet missing;
for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
if (present.find(*i) == present.end()) missing.push_back(*i); if (!present.count(*i)) missing.insert(i->clone());
printMsg(lvlDebug, format("sending %1% missing paths") % missing.size()); printMsg(lvlDebug, format("sending %1% missing paths") % missing.size());
@ -131,7 +132,7 @@ void State::buildRemote(ref<Store> destStore,
{ {
assert(BuildResult::TimedOut == 8); assert(BuildResult::TimedOut == 8);
string base = baseNameOf(step->drvPath); string base(step->drvPath.to_string());
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
AutoDelete autoDelete(result.logFile, false); AutoDelete autoDelete(result.logFile, false);
@ -217,22 +218,22 @@ void State::buildRemote(ref<Store> destStore,
outputs of the input derivations. */ outputs of the input derivations. */
updateStep(ssSendingInputs); updateStep(ssSendingInputs);
PathSet inputs; StorePathSet inputs;
BasicDerivation basicDrv(step->drv); BasicDerivation basicDrv(*step->drv);
if (sendDerivation) if (sendDerivation)
inputs.insert(step->drvPath); inputs.insert(step->drvPath.clone());
else else
for (auto & p : step->drv.inputSrcs) for (auto & p : step->drv->inputSrcs)
inputs.insert(p); inputs.insert(p.clone());
for (auto & input : step->drv.inputDrvs) { for (auto & input : step->drv->inputDrvs) {
Derivation drv2 = readDerivation(input.first); Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
for (auto & name : input.second) { for (auto & name : input.second) {
auto i = drv2.outputs.find(name); auto i = drv2.outputs.find(name);
if (i == drv2.outputs.end()) continue; if (i == drv2.outputs.end()) continue;
inputs.insert(i->second.path); inputs.insert(i->second.path.clone());
basicDrv.inputSrcs.insert(i->second.path); basicDrv.inputSrcs.insert(i->second.path.clone());
} }
} }
@ -241,14 +242,15 @@ void State::buildRemote(ref<Store> destStore,
this will copy the inputs to the binary cache from the local this will copy the inputs to the binary cache from the local
store. */ store. */
if (localStore != std::shared_ptr<Store>(destStore)) if (localStore != std::shared_ptr<Store>(destStore))
copyClosure(ref<Store>(localStore), destStore, step->drv.inputSrcs, NoRepair, NoCheckSigs); copyClosure(ref<Store>(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs);
/* Copy the input closure. */ /* Copy the input closure. */
if (!machine->isLocalhost()) { if (!machine->isLocalhost()) {
auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting); auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
mc1.reset(); mc1.reset();
MaintainCount<counter> mc2(nrStepsCopyingTo); MaintainCount<counter> mc2(nrStepsCopyingTo);
printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName); printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
@ -272,14 +274,19 @@ void State::buildRemote(ref<Store> destStore,
logFD = -1; logFD = -1;
/* Do the build. */ /* Do the build. */
printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName); printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
localStore->printStorePath(step->drvPath),
machine->sshName);
updateStep(ssBuilding); updateStep(ssBuilding);
if (sendDerivation) if (sendDerivation) {
to << cmdBuildPaths << PathSet({step->drvPath}); to << cmdBuildPaths;
else writeStorePaths(*localStore, to, singleton(step->drvPath));
to << cmdBuildDerivation << step->drvPath << basicDrv; } else {
to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
writeDerivation(to, *localStore, basicDrv);
}
to << maxSilentTime << buildTimeout; to << maxSilentTime << buildTimeout;
if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
to << maxLogSize; to << maxLogSize;
@ -380,7 +387,8 @@ void State::buildRemote(ref<Store> destStore,
/* If the path was substituted or already valid, then we didn't /* If the path was substituted or already valid, then we didn't
get a build log. */ get a build log. */
if (result.isCached) { if (result.isCached) {
printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName); printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);
unlink(result.logFile.c_str()); unlink(result.logFile.c_str());
result.logFile = ""; result.logFile = "";
} }
@ -395,13 +403,12 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
PathSet outputs; auto outputs = step->drv->outputPaths();
for (auto & output : step->drv.outputs)
outputs.insert(output.second.path);
/* Query the size of the output paths. */ /* Query the size of the output paths. */
size_t totalNarSize = 0; size_t totalNarSize = 0;
to << cmdQueryPathInfos << outputs; to << cmdQueryPathInfos;
writeStorePaths(*localStore, to, outputs);
to.flush(); to.flush();
while (true) { while (true) {
if (readString(from) == "") break; if (readString(from) == "") break;
@ -416,8 +423,8 @@ void State::buildRemote(ref<Store> destStore,
return; return;
} }
printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)") printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
% step->drvPath % machine->sshName % totalNarSize); localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
/* Block until we have the required amount of memory /* Block until we have the required amount of memory
available, which is twice the NAR size (namely the available, which is twice the NAR size (namely the
@ -431,10 +438,11 @@ void State::buildRemote(ref<Store> destStore,
auto resMs = std::chrono::duration_cast<std::chrono::milliseconds>(resStop - resStart).count(); auto resMs = std::chrono::duration_cast<std::chrono::milliseconds>(resStop - resStart).count();
if (resMs >= 1000) if (resMs >= 1000)
printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s") printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s",
% resMs % totalNarSize % step->drvPath); resMs, totalNarSize, localStore->printStorePath(step->drvPath));
to << cmdExportPaths << 0 << outputs; to << cmdExportPaths << 0;
writeStorePaths(*localStore, to, outputs);
to.flush(); to.flush();
destStore->importPaths(from, result.accessor, NoCheckSigs); destStore->importPaths(from, result.accessor, NoCheckSigs);

28
src/hydra-queue-runner/build-result.cc
View file

@ -14,16 +14,14 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
BuildOutput res; BuildOutput res;
/* Compute the closure size. */ /* Compute the closure size. */
PathSet outputs; auto outputs = drv.outputPaths();
for (auto & output : drv.outputs) StorePathSet closure;
outputs.insert(output.second.path);
PathSet closure;
for (auto & output : outputs) for (auto & output : outputs)
store->computeFSClosure(output, closure); store->computeFSClosure(singleton(output), closure);
for (auto & path : closure) { for (auto & path : closure) {
auto info = store->queryPathInfo(path); auto info = store->queryPathInfo(path);
res.closureSize += info->narSize; res.closureSize += info->narSize;
if (outputs.find(path) != outputs.end()) res.size += info->narSize; if (outputs.count(path)) res.size += info->narSize;
} }
/* Get build products. */ /* Get build products. */
@ -39,11 +37,13 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
, std::regex::extended); , std::regex::extended);
for (auto & output : outputs) { for (auto & output : outputs) {
Path failedFile = output + "/nix-support/failed"; auto outputS = store->printStorePath(output);
Path failedFile = outputS + "/nix-support/failed";
if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)
res.failed = true; res.failed = true;
Path productsFile = output + "/nix-support/hydra-build-products"; Path productsFile = outputS + "/nix-support/hydra-build-products";
if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)
continue; continue;
@ -72,7 +72,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
auto st = accessor->stat(product.path); auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing) continue; if (st.type == FSAccessor::Type::tMissing) continue;
product.name = product.path == output ? "" : baseNameOf(product.path); product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
if (st.type == FSAccessor::Type::tRegular) { if (st.type == FSAccessor::Type::tRegular) {
product.isRegular = true; product.isRegular = true;
@ -91,14 +91,14 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
if (!explicitProducts) { if (!explicitProducts) {
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
BuildProduct product; BuildProduct product;
product.path = output.second.path; product.path = store->printStorePath(output.second.path);
product.type = "nix-build"; product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first; product.subtype = output.first == "out" ? "" : output.first;
product.name = storePathToName(product.path); product.name = output.second.path.name();
auto st = accessor->stat(product.path); auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing) if (st.type == FSAccessor::Type::tMissing)
throw Error(format("getting status of ‘%1%’") % product.path); throw Error("getting status of ‘%s’", product.path);
if (st.type == FSAccessor::Type::tDirectory) if (st.type == FSAccessor::Type::tDirectory)
res.products.push_back(product); res.products.push_back(product);
} }
@ -106,7 +106,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
/* Get the release name from $output/nix-support/hydra-release-name. */ /* Get the release name from $output/nix-support/hydra-release-name. */
for (auto & output : outputs) { for (auto & output : outputs) {
Path p = output + "/nix-support/hydra-release-name"; auto p = store->printStorePath(output) + "/nix-support/hydra-release-name";
if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
try { try {
res.releaseName = trim(accessor->readFile(p)); res.releaseName = trim(accessor->readFile(p));
@ -116,7 +116,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
/* Get metrics. */ /* Get metrics. */
for (auto & output : outputs) { for (auto & output : outputs) {
Path metricsFile = output + "/nix-support/hydra-metrics"; auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics";
if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) { for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line); auto fields = tokenizeString<std::vector<std::string>>(line);

44
src/hydra-queue-runner/builder.cc
View file

@ -18,7 +18,7 @@ void setThreadName(const std::string & name)
void State::builder(MachineReservation::ptr reservation) void State::builder(MachineReservation::ptr reservation)
{ {
setThreadName("bld~" + baseNameOf(reservation->step->drvPath)); setThreadName("bld~" + std::string(reservation->step->drvPath.to_string()));
StepResult res = sRetry; StepResult res = sRetry;
@ -39,8 +39,10 @@ void State::builder(MachineReservation::ptr reservation)
auto destStore = getDestStore(); auto destStore = getDestStore();
res = doBuildStep(destStore, reservation, activeStep); res = doBuildStep(destStore, reservation, activeStep);
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("uncaught exception building ‘%1%’ on ‘%2%’: %3%") printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
% reservation->step->drvPath % reservation->machine->sshName % e.what()); localStore->printStorePath(reservation->step->drvPath),
reservation->machine->sshName,
e.what());
} }
} }
@ -60,7 +62,7 @@ void State::builder(MachineReservation::ptr reservation)
nrRetries++; nrRetries++;
if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic
int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10); int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10);
printMsg(lvlInfo, format("will retry ‘%1%’ after %2%s") % step->drvPath % delta); printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta);
step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta); step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta);
} }
@ -95,7 +97,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
cancelled (namely if there are no more Builds referring to cancelled (namely if there are no more Builds referring to
it). */ it). */
BuildID buildId; BuildID buildId;
Path buildDrvPath; std::optional<StorePath> buildDrvPath;
unsigned int maxSilentTime, buildTimeout; unsigned int maxSilentTime, buildTimeout;
unsigned int repeats = step->isDeterministic ? 1 : 0; unsigned int repeats = step->isDeterministic ? 1 : 0;
@ -116,7 +118,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
possibility, we retry this step (putting it back in possibility, we retry this step (putting it back in
the runnable queue). If there are really no strong the runnable queue). If there are really no strong
pointers to the step, it will be deleted. */ pointers to the step, it will be deleted. */
printMsg(lvlInfo, format("maybe cancelling build step ‘%1%’") % step->drvPath); printMsg(lvlInfo, "maybe cancelling build step ‘%s’", localStore->printStorePath(step->drvPath));
return sMaybeCancelled; return sMaybeCancelled;
} }
@ -138,15 +140,15 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
if (!build) build = *dependents.begin(); if (!build) build = *dependents.begin();
buildId = build->id; buildId = build->id;
buildDrvPath = build->drvPath; buildDrvPath = build->drvPath.clone();
maxSilentTime = build->maxSilentTime; maxSilentTime = build->maxSilentTime;
buildTimeout = build->buildTimeout; buildTimeout = build->buildTimeout;
printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
} }
bool quit = buildId == buildOne && step->drvPath == buildDrvPath; bool quit = buildId == buildOne && step->drvPath == *buildDrvPath;
RemoteResult result; RemoteResult result;
BuildOutput res; BuildOutput res;
@ -166,7 +168,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
try { try {
auto store = destStore.dynamic_pointer_cast<BinaryCacheStore>(); auto store = destStore.dynamic_pointer_cast<BinaryCacheStore>();
if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) { if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) {
store->upsertFile("log/" + baseNameOf(step->drvPath), readFile(result.logFile), "text/plain; charset=utf-8"); store->upsertFile("log/" + std::string(step->drvPath.to_string()), readFile(result.logFile), "text/plain; charset=utf-8");
unlink(result.logFile.c_str()); unlink(result.logFile.c_str());
} }
} catch (...) { } catch (...) {
@ -218,7 +220,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
if (result.stepStatus == bsSuccess) { if (result.stepStatus == bsSuccess) {
updateStep(ssPostProcessing); updateStep(ssPostProcessing);
res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), step->drv); res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), *step->drv);
} }
result.accessor = 0; result.accessor = 0;
@ -255,8 +257,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
/* The step had a hopefully temporary failure (e.g. network /* The step had a hopefully temporary failure (e.g. network
issue). Retry a number of times. */ issue). Retry a number of times. */
if (result.canRetry) { if (result.canRetry) {
printMsg(lvlError, format("possibly transient failure building ‘%1%’ on ‘%2%’: %3%") printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s",
% step->drvPath % machine->sshName % result.errorMsg); localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg);
assert(stepNr); assert(stepNr);
bool retry; bool retry;
{ {
@ -275,7 +277,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr); assert(stepNr);
for (auto & path : step->drv.outputPaths()) for (auto & path : step->drv->outputPaths())
addRoot(path); addRoot(path);
/* Register success in the database for all Build objects that /* Register success in the database for all Build objects that
@ -308,7 +310,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
no new referrers can have been added in the no new referrers can have been added in the
meantime or be added afterwards. */ meantime or be added afterwards. */
if (direct.empty()) { if (direct.empty()) {
printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath); printMsg(lvlDebug, "finishing build step ‘%s’",
localStore->printStorePath(step->drvPath));
steps_->erase(step->drvPath); steps_->erase(step->drvPath);
} }
} }
@ -393,7 +396,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
be certain no new referrers can be added. */ be certain no new referrers can be added. */
if (indirect.empty()) { if (indirect.empty()) {
for (auto & s : steps) { for (auto & s : steps) {
printMsg(lvlDebug, format("finishing build step ‘%1%’") % s->drvPath); printMsg(lvlDebug, "finishing build step ‘%s’",
localStore->printStorePath(s->drvPath));
steps_->erase(s->drvPath); steps_->erase(s->drvPath);
} }
} }
@ -437,8 +441,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
/* Remember failed paths in the database so that they /* Remember failed paths in the database so that they
won't be built again. */ won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache) if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv.outputPaths()) for (auto & path : step->drv->outputPaths())
txn.parameterized("insert into FailedPaths values ($1)")(path).exec(); txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec();
txn.commit(); txn.commit();
} }
@ -478,8 +482,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
} }
void State::addRoot(const Path & storePath) void State::addRoot(const StorePath & storePath)
{ {
auto root = rootsDir + "/" + baseNameOf(storePath); auto root = rootsDir + "/" + std::string(storePath.to_string());
if (!pathExists(root)) writeFile(root, ""); if (!pathExists(root)) writeFile(root, "");
} }

4
src/hydra-queue-runner/dispatcher.cc
View file

@ -10,7 +10,7 @@ using namespace nix;
void State::makeRunnable(Step::ptr step) void State::makeRunnable(Step::ptr step)
{ {
printMsg(lvlChatty, format("step ‘%1%’ is now runnable") % step->drvPath); printMsg(lvlChatty, "step ‘%s’ is now runnable", localStore->printStorePath(step->drvPath));
{ {
auto step_(step->state.lock()); auto step_(step->state.lock());
@ -248,7 +248,7 @@ system_time State::doDispatch()
/* Can this machine do this step? */ /* Can this machine do this step? */
if (!mi.machine->supportsStep(step)) { if (!mi.machine->supportsStep(step)) {
debug("machine '%s' does not support step '%s' (system type '%s')", debug("machine '%s' does not support step '%s' (system type '%s')",
mi.machine->sshName, step->drvPath, step->drv.platform); mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform);
continue; continue;
} }

29
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -37,9 +37,9 @@ static uint64_t getMemSize()
std::string getEnvOrDie(const std::string & key) std::string getEnvOrDie(const std::string & key)
{ {
char * value = getenv(key.c_str()); auto value = getEnv(key);
if (!value) throw Error("environment variable '%s' is not set", key); if (!value) throw Error("environment variable '%s' is not set", key);
return value; return *value;
} }
@ -159,7 +159,7 @@ void State::monitorMachinesFile()
{ {
string defaultMachinesFile = "/etc/nix/machines"; string defaultMachinesFile = "/etc/nix/machines";
auto machinesFiles = tokenizeString<std::vector<Path>>( auto machinesFiles = tokenizeString<std::vector<Path>>(
getEnv("NIX_REMOTE_SYSTEMS", pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");
if (machinesFiles.empty()) { if (machinesFiles.empty()) {
parseMachines("localhost " + parseMachines("localhost " +
@ -251,10 +251,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
(buildId) (buildId)
(stepNr) (stepNr)
(0) // == build (0) // == build
(step->drvPath) (localStore->printStorePath(step->drvPath))
(status == bsBusy ? 1 : 0) (status == bsBusy ? 1 : 0)
(startTime, startTime != 0) (startTime, startTime != 0)
(step->drv.platform) (step->drv->platform)
((int) status, status != bsBusy) ((int) status, status != bsBusy)
(propagatedFrom, propagatedFrom != 0) (propagatedFrom, propagatedFrom != 0)
(errorMsg, errorMsg != "") (errorMsg, errorMsg != "")
@ -263,10 +263,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
if (r.affected_rows() == 0) goto restart; if (r.affected_rows() == 0) goto restart;
for (auto & output : step->drv.outputs) for (auto & output : step->drv->outputs)
txn.parameterized txn.parameterized
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
(buildId)(stepNr)(output.first)(output.second.path).exec(); (buildId)(stepNr)(output.first)(localStore->printStorePath(output.second.path)).exec();
if (status == bsBusy) if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -309,7 +309,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath) Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath)
{ {
restart: restart:
auto stepNr = allocBuildStep(txn, build->id); auto stepNr = allocBuildStep(txn, build->id);
@ -319,7 +319,7 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto
(build->id) (build->id)
(stepNr) (stepNr)
(1) // == substitution (1) // == substitution
(drvPath) (localStore->printStorePath(drvPath))
(0) (0)
(0) (0)
(startTime) (startTime)
@ -329,7 +329,10 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto
txn.parameterized txn.parameterized
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
(build->id)(stepNr)(outputName)(storePath).exec(); (build->id)
(stepNr)
(outputName)
(localStore->printStorePath(storePath)).exec();
return stepNr; return stepNr;
} }
@ -449,8 +452,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn) bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & path : step->drv.outputPaths()) for (auto & path : step->drv->outputPaths())
if (!txn.parameterized("select 1 from FailedPaths where path = $1")(path).exec().empty()) if (!txn.parameterized("select 1 from FailedPaths where path = $1")(localStore->printStorePath(path)).exec().empty())
return true; return true;
return false; return false;
} }
@ -762,7 +765,7 @@ void State::run(BuildID buildOne)
Store::Params localParams; Store::Params localParams;
localParams["max-connections"] = "16"; localParams["max-connections"] = "16";
localParams["max-connection-age"] = "600"; localParams["max-connection-age"] = "600";
localStore = openStore(getEnv("NIX_REMOTE"), localParams); localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams);
auto storeUri = config->getStrOption("store_uri"); auto storeUri = config->getStrOption("store_uri");
_destStore = storeUri == "" ? localStore : openStore(storeUri); _destStore = storeUri == "" ? localStore : openStore(storeUri);

87
src/hydra-queue-runner/queue-monitor.cc
View file

@ -83,7 +83,7 @@ bool State::getQueuedBuilds(Connection & conn,
them yet (since we don't want a long-running transaction). */ them yet (since we don't want a long-running transaction). */
std::vector<BuildID> newIDs; std::vector<BuildID> newIDs;
std::map<BuildID, Build::ptr> newBuildsByID; std::map<BuildID, Build::ptr> newBuildsByID;
std::multimap<Path, BuildID> newBuildsByPath; std::multimap<StorePath, BuildID> newBuildsByPath;
unsigned int newLastBuildId = lastBuildId; unsigned int newLastBuildId = lastBuildId;
@ -104,7 +104,7 @@ bool State::getQueuedBuilds(Connection & conn,
auto build = std::make_shared<Build>(); auto build = std::make_shared<Build>();
build->id = id; build->id = id;
build->drvPath = row["drvPath"].as<string>(); build->drvPath = localStore->parseStorePath(row["drvPath"].as<string>());
build->projectName = row["project"].as<string>(); build->projectName = row["project"].as<string>();
build->jobsetName = row["jobset"].as<string>(); build->jobsetName = row["jobset"].as<string>();
build->jobName = row["job"].as<string>(); build->jobName = row["job"].as<string>();
@ -117,14 +117,14 @@ bool State::getQueuedBuilds(Connection & conn,
newIDs.push_back(id); newIDs.push_back(id);
newBuildsByID[id] = build; newBuildsByID[id] = build;
newBuildsByPath.emplace(std::make_pair(build->drvPath, id)); newBuildsByPath.emplace(std::make_pair(build->drvPath.clone(), id));
} }
} }
std::set<Step::ptr> newRunnable; std::set<Step::ptr> newRunnable;
unsigned int nrAdded; unsigned int nrAdded;
std::function<void(Build::ptr)> createBuild; std::function<void(Build::ptr)> createBuild;
std::set<Path> finishedDrvs; std::set<StorePath> finishedDrvs;
createBuild = [&](Build::ptr build) { createBuild = [&](Build::ptr build) {
printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
@ -160,7 +160,8 @@ bool State::getQueuedBuilds(Connection & conn,
/* Some step previously failed, so mark the build as /* Some step previously failed, so mark the build as
failed right away. */ failed right away. */
printMsg(lvlError, format("marking build %d as cached failure due to ‘%s’") % build->id % ex.step->drvPath); printMsg(lvlError, "marking build %d as cached failure due to ‘%s’",
build->id, localStore->printStorePath(ex.step->drvPath));
if (!build->finishedInDB) { if (!build->finishedInDB) {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(conn); pqxx::work txn(conn);
@ -171,14 +172,14 @@ bool State::getQueuedBuilds(Connection & conn,
auto res = txn.parameterized auto res = txn.parameterized
("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1") ("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1")
(ex.step->drvPath).exec(); (localStore->printStorePath(ex.step->drvPath)).exec();
if (!res[0][0].is_null()) propagatedFrom = res[0][0].as<BuildID>(); if (!res[0][0].is_null()) propagatedFrom = res[0][0].as<BuildID>();
if (!propagatedFrom) { if (!propagatedFrom) {
for (auto & output : ex.step->drv.outputs) { for (auto & output : ex.step->drv->outputs) {
auto res = txn.parameterized auto res = txn.parameterized
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1") ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1")
(output.second.path).exec(); (localStore->printStorePath(output.second.path)).exec();
if (!res[0][0].is_null()) { if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>(); propagatedFrom = res[0][0].as<BuildID>();
break; break;
@ -217,7 +218,7 @@ bool State::getQueuedBuilds(Connection & conn,
/* If we didn't get a step, it means the step's outputs are /* If we didn't get a step, it means the step's outputs are
all valid. So we mark this as a finished, cached build. */ all valid. So we mark this as a finished, cached build. */
if (!step) { if (!step) {
Derivation drv = readDerivation(build->drvPath); Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath));
BuildOutput res = getBuildOutputCached(conn, destStore, drv); BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths()) for (auto & path : drv.outputPaths())
@ -227,7 +228,7 @@ bool State::getQueuedBuilds(Connection & conn,
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(conn); pqxx::work txn(conn);
time_t now = time(0); time_t now = time(0);
printMsg(lvlInfo, format("marking build %1% as succeeded (cached)") % build->id); printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id);
markSucceededBuild(txn, build, res, true, now, now); markSucceededBuild(txn, build, res, true, now, now);
notifyBuildFinished(txn, build->id, {}); notifyBuildFinished(txn, build->id, {});
txn.commit(); txn.commit();
@ -250,8 +251,8 @@ bool State::getQueuedBuilds(Connection & conn,
build->propagatePriorities(); build->propagatePriorities();
printMsg(lvlChatty, format("added build %1% (top-level step %2%, %3% new steps)") printMsg(lvlChatty, "added build %1% (top-level step %2%, %3% new steps)",
% build->id % step->drvPath % newSteps.size()); build->id, localStore->printStorePath(step->drvPath), newSteps.size());
}; };
/* Now instantiate build steps for each new build. The builder /* Now instantiate build steps for each new build. The builder
@ -271,7 +272,7 @@ bool State::getQueuedBuilds(Connection & conn,
try { try {
createBuild(build); createBuild(build);
} catch (Error & e) { } catch (Error & e) {
e.addPrefix(format("while loading build %1%: ") % build->id); e.addPrefix(fmt("while loading build %1%: ", build->id));
throw; throw;
} }
@ -358,10 +359,12 @@ void State::processQueueChange(Connection & conn)
activeStepState->cancelled = true; activeStepState->cancelled = true;
if (activeStepState->pid != -1) { if (activeStepState->pid != -1) {
printInfo("killing builder process %d of build step ‘%s’", printInfo("killing builder process %d of build step ‘%s’",
activeStepState->pid, activeStep->step->drvPath); activeStepState->pid,
localStore->printStorePath(activeStep->step->drvPath));
if (kill(activeStepState->pid, SIGINT) == -1) if (kill(activeStepState->pid, SIGINT) == -1)
printError("error killing build step ‘%s’: %s", printError("error killing build step ‘%s’: %s",
activeStep->step->drvPath, strerror(errno)); localStore->printStorePath(activeStep->step->drvPath),
strerror(errno));
} }
} }
} }
@ -370,8 +373,8 @@ void State::processQueueChange(Connection & conn)
Step::ptr State::createStep(ref<Store> destStore, Step::ptr State::createStep(ref<Store> destStore,
Connection & conn, Build::ptr build, const Path & drvPath, Connection & conn, Build::ptr build, const StorePath & drvPath,
Build::ptr referringBuild, Step::ptr referringStep, std::set<Path> & finishedDrvs, Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable) std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable)
{ {
if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0; if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0;
@ -400,7 +403,7 @@ Step::ptr State::createStep(ref<Store> destStore,
/* If it doesn't exist, create it. */ /* If it doesn't exist, create it. */
if (!step) { if (!step) {
step = std::make_shared<Step>(); step = std::make_shared<Step>();
step->drvPath = drvPath; step->drvPath = drvPath.clone();
isNew = true; isNew = true;
} }
@ -414,28 +417,28 @@ Step::ptr State::createStep(ref<Store> destStore,
if (referringStep) if (referringStep)
step_->rdeps.push_back(referringStep); step_->rdeps.push_back(referringStep);
(*steps_)[drvPath] = step; steps_->insert_or_assign(drvPath.clone(), step);
} }
if (!isNew) return step; if (!isNew) return step;
printMsg(lvlDebug, format("considering derivation ‘%1%’") % drvPath); printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath));
/* Initialize the step. Note that the step may be visible in /* Initialize the step. Note that the step may be visible in
‘steps’ before this point, but that doesn't matter because ‘steps’ before this point, but that doesn't matter because
it's not runnable yet, and other threads won't make it it's not runnable yet, and other threads won't make it
runnable while step->created == false. */ runnable while step->created == false. */
step->drv = readDerivation(drvPath); step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath)));
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, step->drv); step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *step->drv);
step->preferLocalBuild = step->parsedDrv->willBuildLocally(); step->preferLocalBuild = step->parsedDrv->willBuildLocally();
step->isDeterministic = get(step->drv.env, "isDetermistic", "0") == "1"; step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1";
step->systemType = step->drv.platform; step->systemType = step->drv->platform;
{ {
auto i = step->drv.env.find("requiredSystemFeatures"); auto i = step->drv->env.find("requiredSystemFeatures");
StringSet features; StringSet features;
if (i != step->drv.env.end()) if (i != step->drv->env.end())
features = step->requiredSystemFeatures = tokenizeString<std::set<std::string>>(i->second); features = step->requiredSystemFeatures = tokenizeString<std::set<std::string>>(i->second);
if (step->preferLocalBuild) if (step->preferLocalBuild)
features.insert("local"); features.insert("local");
@ -451,12 +454,13 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */ /* Are all outputs valid? */
bool valid = true; bool valid = true;
PathSet outputs = step->drv.outputPaths(); auto outputs = step->drv->outputPaths();
DerivationOutputs missing; DerivationOutputs missing;
for (auto & i : step->drv.outputs) for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path)) { if (!destStore->isValidPath(i.second.path)) {
valid = false; valid = false;
missing[i.first] = i.second; missing.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
} }
/* Try to copy the missing paths from the local store or from /* Try to copy the missing paths from the local store or from
@ -469,7 +473,7 @@ Step::ptr State::createStep(ref<Store> destStore,
avail++; avail++;
else if (useSubstitutes) { else if (useSubstitutes) {
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({i.second.path}, infos); localStore->querySubstitutablePathInfos(singleton(i.second.path), infos);
if (infos.size() == 1) if (infos.size() == 1)
avail++; avail++;
} }
@ -482,14 +486,18 @@ Step::ptr State::createStep(ref<Store> destStore,
time_t startTime = time(0); time_t startTime = time(0);
if (localStore->isValidPath(i.second.path)) if (localStore->isValidPath(i.second.path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store", i.second.path, drvPath); printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(i.second.path),
localStore->printStorePath(drvPath));
else { else {
printInfo("substituting output ‘%1%’ of ‘%2%’", i.second.path, drvPath); printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(i.second.path),
localStore->printStorePath(drvPath));
localStore->ensurePath(i.second.path); localStore->ensurePath(i.second.path);
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
copyClosure(ref<Store>(localStore), destStore, {i.second.path}); copyClosure(ref<Store>(localStore), destStore, singleton(i.second.path));
time_t stopTime = time(0); time_t stopTime = time(0);
@ -501,7 +509,10 @@ Step::ptr State::createStep(ref<Store> destStore,
} }
} catch (Error & e) { } catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s", i.second.path, drvPath, e.what()); printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(i.second.path),
localStore->printStorePath(drvPath),
e.what());
valid = false; valid = false;
break; break;
} }
@ -511,15 +522,15 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: check whether all outputs are in the binary cache. // FIXME: check whether all outputs are in the binary cache.
if (valid) { if (valid) {
finishedDrvs.insert(drvPath); finishedDrvs.insert(drvPath.clone());
return 0; return 0;
} }
/* No, we need to build. */ /* No, we need to build. */
printMsg(lvlDebug, format("creating build step ‘%1%’") % drvPath); printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath));
/* Create steps for the dependencies. */ /* Create steps for the dependencies. */
for (auto & i : step->drv.inputDrvs) { for (auto & i : step->drv->inputDrvs) {
auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
if (dep) { if (dep) {
auto step_(step->state.lock()); auto step_(step->state.lock());
@ -607,7 +618,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
("select id, buildStatus, releaseName, closureSize, size from Builds b " ("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build " "join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1") "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1")
(output.second.path).exec(); (localStore->printStorePath(output.second.path)).exec();
if (r.empty()) continue; if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>(); BuildID id = r[0][0].as<BuildID>();

20
src/hydra-queue-runner/state.hh
View file

@ -123,8 +123,8 @@ struct Build
typedef std::weak_ptr<Build> wptr; typedef std::weak_ptr<Build> wptr;
BuildID id; BuildID id;
nix::Path drvPath; nix::StorePath drvPath;
std::map<std::string, nix::Path> outputs; std::map<std::string, nix::StorePath> outputs;
std::string projectName, jobsetName, jobName; std::string projectName, jobsetName, jobName;
time_t timestamp; time_t timestamp;
unsigned int maxSilentTime, buildTimeout; unsigned int maxSilentTime, buildTimeout;
@ -150,8 +150,8 @@ struct Step
typedef std::shared_ptr<Step> ptr; typedef std::shared_ptr<Step> ptr;
typedef std::weak_ptr<Step> wptr; typedef std::weak_ptr<Step> wptr;
nix::Path drvPath; nix::StorePath drvPath;
nix::Derivation drv; std::unique_ptr<nix::Derivation> drv;
std::unique_ptr<nix::ParsedDerivation> parsedDrv; std::unique_ptr<nix::ParsedDerivation> parsedDrv;
std::set<std::string> requiredSystemFeatures; std::set<std::string> requiredSystemFeatures;
bool preferLocalBuild; bool preferLocalBuild;
@ -252,7 +252,7 @@ struct Machine
{ {
/* Check that this machine is of the type required by the /* Check that this machine is of the type required by the
step. */ step. */
if (!systemTypes.count(step->drv.platform == "builtin" ? nix::settings.thisSystem : step->drv.platform)) if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
return false; return false;
/* Check that the step requires all mandatory features of this /* Check that the step requires all mandatory features of this
@ -313,7 +313,7 @@ private:
queued builds). Note that these are weak pointers. Steps are queued builds). Note that these are weak pointers. Steps are
kept alive by being reachable from Builds or by being in kept alive by being reachable from Builds or by being in
progress. */ progress. */
typedef std::map<nix::Path, Step::wptr> Steps; typedef std::map<nix::StorePath, Step::wptr> Steps;
nix::Sync<Steps> steps; nix::Sync<Steps> steps;
/* Build steps that have no unbuilt dependencies. */ /* Build steps that have no unbuilt dependencies. */
@ -454,7 +454,7 @@ private:
const std::string & machine); const std::string & machine);
int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath); Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath);
void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
@ -473,8 +473,8 @@ private:
const nix::Derivation & drv); const nix::Derivation & drv);
Step::ptr createStep(nix::ref<nix::Store> store, Step::ptr createStep(nix::ref<nix::Store> store,
Connection & conn, Build::ptr build, const nix::Path & drvPath, Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::Path> & finishedDrvs, Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable); std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable);
Jobset::ptr createJobset(pqxx::work & txn, Jobset::ptr createJobset(pqxx::work & txn,
@ -523,7 +523,7 @@ private:
void dumpStatus(Connection & conn, bool log); void dumpStatus(Connection & conn, bool log);
void addRoot(const nix::Path & storePath); void addRoot(const nix::StorePath & storePath);
public: public:

2
src/hydra-queue-runner/token-server.hh
View file

@ -7,7 +7,7 @@
namespace nix { namespace nix {
MakeError(NoTokens, Error) MakeError(NoTokens, Error);
/* This class hands out tokens. There are only ‘maxTokens’ tokens /* This class hands out tokens. There are only ‘maxTokens’ tokens
available. Calling get(N) will return a Token object, representing available. Calling get(N) will return a Token object, representing

2
src/libhydra/db.hh
View file

@ -12,7 +12,7 @@ struct Connection : pqxx::connection
std::string getFlags() std::string getFlags()
{ {
using namespace nix; using namespace nix;
auto s = getEnv("HYDRA_DBI", "dbi:Pg:dbname=hydra;"); auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;");
std::string prefix = "dbi:Pg:"; std::string prefix = "dbi:Pg:";
if (std::string(s, 0, prefix.size()) != prefix) if (std::string(s, 0, prefix.size()) != prefix)
throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); throw Error("$HYDRA_DBI does not denote a PostgreSQL database");

4
src/libhydra/hydra-config.hh
View file

@ -14,9 +14,9 @@ struct Config
/* Read hydra.conf. */ /* Read hydra.conf. */
auto hydraConfigFile = getEnv("HYDRA_CONFIG"); auto hydraConfigFile = getEnv("HYDRA_CONFIG");
if (pathExists(hydraConfigFile)) { if (hydraConfigFile && pathExists(*hydraConfigFile)) {
for (auto line : tokenizeString<Strings>(readFile(hydraConfigFile), "\n")) { for (auto line : tokenizeString<Strings>(readFile(*hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#'))); line = trim(string(line, 0, line.find('#')));
auto eq = line.find('='); auto eq = line.find('=');