From e7f2139e251cb73195eea6fb84e2a6167b4db968 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Dec 2019 22:49:26 +0100 Subject: [PATCH] Build against nix-master --- flake.lock | 12 +-- src/hydra-eval-jobs/Makefile.am | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +- src/hydra-queue-runner/Makefile.am | 2 +- src/hydra-queue-runner/build-remote.cc | 76 +++++++++-------- src/hydra-queue-runner/build-result.cc | 28 +++---- src/hydra-queue-runner/builder.cc | 44 +++++----- src/hydra-queue-runner/dispatcher.cc | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 29 ++++--- src/hydra-queue-runner/queue-monitor.cc | 87 +++++++++++--------- src/hydra-queue-runner/state.hh | 20 ++--- src/hydra-queue-runner/token-server.hh | 2 +- src/libhydra/db.hh | 2 +- src/libhydra/hydra-config.hh | 4 +- 14 files changed, 172 insertions(+), 145 deletions(-) diff --git a/flake.lock b/flake.lock index 93811e5e..e96b9e43 100644 --- a/flake.lock +++ b/flake.lock @@ -4,20 +4,20 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", + "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" + "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" } }, - "narHash": "sha256-zfFjX4SmTKGponLg7X1Lc4XHOUNEDz3czSjGaZTHXOk=", + "narHash": "sha256-wiOmdFFMhuBGEcAURvj7c1LAcVbO82QjcFr+5WjuNzM=", "originalUrl": "nix", - "url": "github:NixOS/nix/2c1e05ae9389742dac637a6f051f718397eff2db" + "url": "github:NixOS/nix/c7866733d7ce2836fbb43de90dd64d17b0d20753" }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", + "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" + "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" } }, "version": 3 diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am index 7a4e9c91..b41a4eb8 100644 --- a/src/hydra-eval-jobs/Makefile.am +++ b/src/hydra-eval-jobs/Makefile.am @@ -1,5 +1,5 @@ bin_PROGRAMS = hydra-eval-jobs hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc -hydra_eval_jobs_LDADD = $(NIX_LIBS) +hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index a309abc7..334864d5 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -115,8 +115,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, done. */ auto localStore = state.store.dynamic_pointer_cast(); if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + baseNameOf(drvPath); - if (!pathExists(root)) localStore->addPermRoot(drvPath, root, false); + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); } auto res2 = res.object("outputs"); diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 1726d0df..af95a3f9 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh token-server.hh state.hh db.hh -hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx +hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 81692849..6070bd4e 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -82,10 +82,10 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, - FdSource & from, FdSink & to, const PathSet & paths, + FdSource & from, FdSink & to, const StorePathSet & paths, bool useSubstitutes = false) { - PathSet closure; + StorePathSet closure; for (auto & path : paths) destStore->computeFSClosure(path, closure); @@ -94,20 +94,21 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << cmdQueryValidPaths << 1 << useSubstitutes << closure; + to << cmdQueryValidPaths << 1 << useSubstitutes; + writeStorePaths(*destStore, to, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = readStorePaths(*destStore, from); + auto present = readStorePaths(*destStore, from); if (present.size() == closure.size()) return; - Paths sorted = destStore->topoSortPaths(closure); + auto sorted = destStore->topoSortPaths(closure); - Paths missing; + StorePathSet missing; for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (present.find(*i) == present.end()) missing.push_back(*i); + if (!present.count(*i)) missing.insert(i->clone()); printMsg(lvlDebug, format("sending %1% missing paths") % missing.size()); @@ -131,7 +132,7 @@ void State::buildRemote(ref destStore, { assert(BuildResult::TimedOut == 8); - string base = baseNameOf(step->drvPath); + string base(step->drvPath.to_string()); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); AutoDelete autoDelete(result.logFile, false); @@ -217,22 +218,22 @@ void State::buildRemote(ref destStore, outputs of the input derivations. */ updateStep(ssSendingInputs); - PathSet inputs; - BasicDerivation basicDrv(step->drv); + StorePathSet inputs; + BasicDerivation basicDrv(*step->drv); if (sendDerivation) - inputs.insert(step->drvPath); + inputs.insert(step->drvPath.clone()); else - for (auto & p : step->drv.inputSrcs) - inputs.insert(p); + for (auto & p : step->drv->inputSrcs) + inputs.insert(p.clone()); - for (auto & input : step->drv.inputDrvs) { - Derivation drv2 = readDerivation(input.first); + for (auto & input : step->drv->inputDrvs) { + Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); for (auto & name : input.second) { auto i = drv2.outputs.find(name); if (i == drv2.outputs.end()) continue; - inputs.insert(i->second.path); - basicDrv.inputSrcs.insert(i->second.path); + inputs.insert(i->second.path.clone()); + basicDrv.inputSrcs.insert(i->second.path.clone()); } } @@ -241,14 +242,15 @@ void State::buildRemote(ref destStore, this will copy the inputs to the binary cache from the local store. */ if (localStore != std::shared_ptr(destStore)) - copyClosure(ref(localStore), destStore, step->drv.inputSrcs, NoRepair, NoCheckSigs); + copyClosure(ref(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs); /* Copy the input closure. */ if (!machine->isLocalhost()) { auto mc1 = std::make_shared>(nrStepsWaiting); mc1.reset(); MaintainCount mc2(nrStepsCopyingTo); - printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); auto now1 = std::chrono::steady_clock::now(); @@ -272,14 +274,19 @@ void State::buildRemote(ref destStore, logFD = -1; /* Do the build. */ - printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "building ‘%s’ on ‘%s’", + localStore->printStorePath(step->drvPath), + machine->sshName); updateStep(ssBuilding); - if (sendDerivation) - to << cmdBuildPaths << PathSet({step->drvPath}); - else - to << cmdBuildDerivation << step->drvPath << basicDrv; + if (sendDerivation) { + to << cmdBuildPaths; + writeStorePaths(*localStore, to, singleton(step->drvPath)); + } else { + to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + writeDerivation(to, *localStore, basicDrv); + } to << maxSilentTime << buildTimeout; if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) to << maxLogSize; @@ -380,7 +387,8 @@ void State::buildRemote(ref destStore, /* If the path was substituted or already valid, then we didn't get a build log. */ if (result.isCached) { - printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); unlink(result.logFile.c_str()); result.logFile = ""; } @@ -395,13 +403,12 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - PathSet outputs; - for (auto & output : step->drv.outputs) - outputs.insert(output.second.path); + auto outputs = step->drv->outputPaths(); /* Query the size of the output paths. */ size_t totalNarSize = 0; - to << cmdQueryPathInfos << outputs; + to << cmdQueryPathInfos; + writeStorePaths(*localStore, to, outputs); to.flush(); while (true) { if (readString(from) == "") break; @@ -416,8 +423,8 @@ void State::buildRemote(ref destStore, return; } - printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)") - % step->drvPath % machine->sshName % totalNarSize); + printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", + localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); /* Block until we have the required amount of memory available, which is twice the NAR size (namely the @@ -431,10 +438,11 @@ void State::buildRemote(ref destStore, auto resMs = std::chrono::duration_cast(resStop - resStart).count(); if (resMs >= 1000) - printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s") - % resMs % totalNarSize % step->drvPath); + printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s", + resMs, totalNarSize, localStore->printStorePath(step->drvPath)); - to << cmdExportPaths << 0 << outputs; + to << cmdExportPaths << 0; + writeStorePaths(*localStore, to, outputs); to.flush(); destStore->importPaths(from, result.accessor, NoCheckSigs); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 25e8c41a..cd8f0a39 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -14,16 +14,14 @@ BuildOutput getBuildOutput(nix::ref store, BuildOutput res; /* Compute the closure size. */ - PathSet outputs; - for (auto & output : drv.outputs) - outputs.insert(output.second.path); - PathSet closure; + auto outputs = drv.outputPaths(); + StorePathSet closure; for (auto & output : outputs) - store->computeFSClosure(output, closure); + store->computeFSClosure(singleton(output), closure); for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; - if (outputs.find(path) != outputs.end()) res.size += info->narSize; + if (outputs.count(path)) res.size += info->narSize; } /* Get build products. */ @@ -39,11 +37,13 @@ BuildOutput getBuildOutput(nix::ref store, , std::regex::extended); for (auto & output : outputs) { - Path failedFile = output + "/nix-support/failed"; + auto outputS = store->printStorePath(output); + + Path failedFile = outputS + "/nix-support/failed"; if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; + Path productsFile = outputS + "/nix-support/hydra-build-products"; if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) continue; @@ -72,7 +72,7 @@ BuildOutput getBuildOutput(nix::ref store, auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) continue; - product.name = product.path == output ? "" : baseNameOf(product.path); + product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); if (st.type == FSAccessor::Type::tRegular) { product.isRegular = true; @@ -91,14 +91,14 @@ BuildOutput getBuildOutput(nix::ref store, if (!explicitProducts) { for (auto & output : drv.outputs) { BuildProduct product; - product.path = output.second.path; + product.path = store->printStorePath(output.second.path); product.type = "nix-build"; product.subtype = output.first == "out" ? "" : output.first; - product.name = storePathToName(product.path); + product.name = output.second.path.name(); auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("getting status of ‘%1%’") % product.path); + throw Error("getting status of ‘%s’", product.path); if (st.type == FSAccessor::Type::tDirectory) res.products.push_back(product); } @@ -106,7 +106,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get the release name from $output/nix-support/hydra-release-name. */ for (auto & output : outputs) { - Path p = output + "/nix-support/hydra-release-name"; + auto p = store->printStorePath(output) + "/nix-support/hydra-release-name"; if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; try { res.releaseName = trim(accessor->readFile(p)); @@ -116,7 +116,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get metrics. */ for (auto & output : outputs) { - Path metricsFile = output + "/nix-support/hydra-metrics"; + auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics"; if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; for (auto & line : tokenizeString(accessor->readFile(metricsFile), "\n")) { auto fields = tokenizeString>(line); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index edd4b1f7..a0fc01c3 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -18,7 +18,7 @@ void setThreadName(const std::string & name) void State::builder(MachineReservation::ptr reservation) { - setThreadName("bld~" + baseNameOf(reservation->step->drvPath)); + setThreadName("bld~" + std::string(reservation->step->drvPath.to_string())); StepResult res = sRetry; @@ -39,8 +39,10 @@ void State::builder(MachineReservation::ptr reservation) auto destStore = getDestStore(); res = doBuildStep(destStore, reservation, activeStep); } catch (std::exception & e) { - printMsg(lvlError, format("uncaught exception building ‘%1%’ on ‘%2%’: %3%") - % reservation->step->drvPath % reservation->machine->sshName % e.what()); + printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(reservation->step->drvPath), + reservation->machine->sshName, + e.what()); } } @@ -60,7 +62,7 @@ void State::builder(MachineReservation::ptr reservation) nrRetries++; if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10); - printMsg(lvlInfo, format("will retry ‘%1%’ after %2%s") % step->drvPath % delta); + printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta); step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta); } @@ -95,7 +97,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, cancelled (namely if there are no more Builds referring to it). */ BuildID buildId; - Path buildDrvPath; + std::optional buildDrvPath; unsigned int maxSilentTime, buildTimeout; unsigned int repeats = step->isDeterministic ? 1 : 0; @@ -116,7 +118,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, possibility, we retry this step (putting it back in the runnable queue). If there are really no strong pointers to the step, it will be deleted. */ - printMsg(lvlInfo, format("maybe cancelling build step ‘%1%’") % step->drvPath); + printMsg(lvlInfo, "maybe cancelling build step ‘%s’", localStore->printStorePath(step->drvPath)); return sMaybeCancelled; } @@ -138,15 +140,15 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (!build) build = *dependents.begin(); buildId = build->id; - buildDrvPath = build->drvPath; + buildDrvPath = build->drvPath.clone(); maxSilentTime = build->maxSilentTime; buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } - bool quit = buildId == buildOne && step->drvPath == buildDrvPath; + bool quit = buildId == buildOne && step->drvPath == *buildDrvPath; RemoteResult result; BuildOutput res; @@ -166,7 +168,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, try { auto store = destStore.dynamic_pointer_cast(); if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) { - store->upsertFile("log/" + baseNameOf(step->drvPath), readFile(result.logFile), "text/plain; charset=utf-8"); + store->upsertFile("log/" + std::string(step->drvPath.to_string()), readFile(result.logFile), "text/plain; charset=utf-8"); unlink(result.logFile.c_str()); } } catch (...) { @@ -218,7 +220,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, ref(result.accessor), step->drv); + res = getBuildOutput(destStore, ref(result.accessor), *step->drv); } result.accessor = 0; @@ -255,8 +257,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* The step had a hopefully temporary failure (e.g. network issue). Retry a number of times. */ if (result.canRetry) { - printMsg(lvlError, format("possibly transient failure building ‘%1%’ on ‘%2%’: %3%") - % step->drvPath % machine->sshName % result.errorMsg); + printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg); assert(stepNr); bool retry; { @@ -275,7 +277,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & path : step->drv.outputPaths()) + for (auto & path : step->drv->outputPaths()) addRoot(path); /* Register success in the database for all Build objects that @@ -308,7 +310,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, no new referrers can have been added in the meantime or be added afterwards. */ if (direct.empty()) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(step->drvPath)); steps_->erase(step->drvPath); } } @@ -393,7 +396,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, be certain no new referrers can be added. */ if (indirect.empty()) { for (auto & s : steps) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % s->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(s->drvPath)); steps_->erase(s->drvPath); } } @@ -437,8 +441,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* Remember failed paths in the database so that they won't be built again. */ if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv.outputPaths()) - txn.parameterized("insert into FailedPaths values ($1)")(path).exec(); + for (auto & path : step->drv->outputPaths()) + txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec(); txn.commit(); } @@ -478,8 +482,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, } -void State::addRoot(const Path & storePath) +void State::addRoot(const StorePath & storePath) { - auto root = rootsDir + "/" + baseNameOf(storePath); + auto root = rootsDir + "/" + std::string(storePath.to_string()); if (!pathExists(root)) writeFile(root, ""); } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 068d5c57..86596ff5 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -10,7 +10,7 @@ using namespace nix; void State::makeRunnable(Step::ptr step) { - printMsg(lvlChatty, format("step ‘%1%’ is now runnable") % step->drvPath); + printMsg(lvlChatty, "step ‘%s’ is now runnable", localStore->printStorePath(step->drvPath)); { auto step_(step->state.lock()); @@ -248,7 +248,7 @@ system_time State::doDispatch() /* Can this machine do this step? */ if (!mi.machine->supportsStep(step)) { debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->sshName, step->drvPath, step->drv.platform); + mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform); continue; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index fa7515d3..5a30c08e 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -37,9 +37,9 @@ static uint64_t getMemSize() std::string getEnvOrDie(const std::string & key) { - char * value = getenv(key.c_str()); + auto value = getEnv(key); if (!value) throw Error("environment variable '%s' is not set", key); - return value; + return *value; } @@ -159,7 +159,7 @@ void State::monitorMachinesFile() { string defaultMachinesFile = "/etc/nix/machines"; auto machinesFiles = tokenizeString>( - getEnv("NIX_REMOTE_SYSTEMS", pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); + getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); if (machinesFiles.empty()) { parseMachines("localhost " + @@ -251,10 +251,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID (buildId) (stepNr) (0) // == build - (step->drvPath) + (localStore->printStorePath(step->drvPath)) (status == bsBusy ? 1 : 0) (startTime, startTime != 0) - (step->drv.platform) + (step->drv->platform) ((int) status, status != bsBusy) (propagatedFrom, propagatedFrom != 0) (errorMsg, errorMsg != "") @@ -263,10 +263,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & output : step->drv.outputs) + for (auto & output : step->drv->outputs) txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (buildId)(stepNr)(output.first)(output.second.path).exec(); + (buildId)(stepNr)(output.first)(localStore->printStorePath(output.second.path)).exec(); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -309,7 +309,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath) + Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath) { restart: auto stepNr = allocBuildStep(txn, build->id); @@ -319,7 +319,7 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto (build->id) (stepNr) (1) // == substitution - (drvPath) + (localStore->printStorePath(drvPath)) (0) (0) (startTime) @@ -329,7 +329,10 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (build->id)(stepNr)(outputName)(storePath).exec(); + (build->id) + (stepNr) + (outputName) + (localStore->printStorePath(storePath)).exec(); return stepNr; } @@ -449,8 +452,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, bool State::checkCachedFailure(Step::ptr step, Connection & conn) { pqxx::work txn(conn); - for (auto & path : step->drv.outputPaths()) - if (!txn.parameterized("select 1 from FailedPaths where path = $1")(path).exec().empty()) + for (auto & path : step->drv->outputPaths()) + if (!txn.parameterized("select 1 from FailedPaths where path = $1")(localStore->printStorePath(path)).exec().empty()) return true; return false; } @@ -762,7 +765,7 @@ void State::run(BuildID buildOne) Store::Params localParams; localParams["max-connections"] = "16"; localParams["max-connection-age"] = "600"; - localStore = openStore(getEnv("NIX_REMOTE"), localParams); + localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams); auto storeUri = config->getStrOption("store_uri"); _destStore = storeUri == "" ? localStore : openStore(storeUri); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index e657a4b8..62de134a 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -83,7 +83,7 @@ bool State::getQueuedBuilds(Connection & conn, them yet (since we don't want a long-running transaction). */ std::vector newIDs; std::map newBuildsByID; - std::multimap newBuildsByPath; + std::multimap newBuildsByPath; unsigned int newLastBuildId = lastBuildId; @@ -104,7 +104,7 @@ bool State::getQueuedBuilds(Connection & conn, auto build = std::make_shared(); build->id = id; - build->drvPath = row["drvPath"].as(); + build->drvPath = localStore->parseStorePath(row["drvPath"].as()); build->projectName = row["project"].as(); build->jobsetName = row["jobset"].as(); build->jobName = row["job"].as(); @@ -117,14 +117,14 @@ bool State::getQueuedBuilds(Connection & conn, newIDs.push_back(id); newBuildsByID[id] = build; - newBuildsByPath.emplace(std::make_pair(build->drvPath, id)); + newBuildsByPath.emplace(std::make_pair(build->drvPath.clone(), id)); } } std::set newRunnable; unsigned int nrAdded; std::function createBuild; - std::set finishedDrvs; + std::set finishedDrvs; createBuild = [&](Build::ptr build) { printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); @@ -160,7 +160,8 @@ bool State::getQueuedBuilds(Connection & conn, /* Some step previously failed, so mark the build as failed right away. */ - printMsg(lvlError, format("marking build %d as cached failure due to ‘%s’") % build->id % ex.step->drvPath); + printMsg(lvlError, "marking build %d as cached failure due to ‘%s’", + build->id, localStore->printStorePath(ex.step->drvPath)); if (!build->finishedInDB) { auto mc = startDbUpdate(); pqxx::work txn(conn); @@ -171,14 +172,14 @@ bool State::getQueuedBuilds(Connection & conn, auto res = txn.parameterized ("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1") - (ex.step->drvPath).exec(); + (localStore->printStorePath(ex.step->drvPath)).exec(); if (!res[0][0].is_null()) propagatedFrom = res[0][0].as(); if (!propagatedFrom) { - for (auto & output : ex.step->drv.outputs) { + for (auto & output : ex.step->drv->outputs) { auto res = txn.parameterized ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -217,7 +218,7 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - Derivation drv = readDerivation(build->drvPath); + Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath)); BuildOutput res = getBuildOutputCached(conn, destStore, drv); for (auto & path : drv.outputPaths()) @@ -227,7 +228,7 @@ bool State::getQueuedBuilds(Connection & conn, auto mc = startDbUpdate(); pqxx::work txn(conn); time_t now = time(0); - printMsg(lvlInfo, format("marking build %1% as succeeded (cached)") % build->id); + printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id); markSucceededBuild(txn, build, res, true, now, now); notifyBuildFinished(txn, build->id, {}); txn.commit(); @@ -250,8 +251,8 @@ bool State::getQueuedBuilds(Connection & conn, build->propagatePriorities(); - printMsg(lvlChatty, format("added build %1% (top-level step %2%, %3% new steps)") - % build->id % step->drvPath % newSteps.size()); + printMsg(lvlChatty, "added build %1% (top-level step %2%, %3% new steps)", + build->id, localStore->printStorePath(step->drvPath), newSteps.size()); }; /* Now instantiate build steps for each new build. The builder @@ -271,7 +272,7 @@ bool State::getQueuedBuilds(Connection & conn, try { createBuild(build); } catch (Error & e) { - e.addPrefix(format("while loading build %1%: ") % build->id); + e.addPrefix(fmt("while loading build %1%: ", build->id)); throw; } @@ -358,10 +359,12 @@ void State::processQueueChange(Connection & conn) activeStepState->cancelled = true; if (activeStepState->pid != -1) { printInfo("killing builder process %d of build step ‘%s’", - activeStepState->pid, activeStep->step->drvPath); + activeStepState->pid, + localStore->printStorePath(activeStep->step->drvPath)); if (kill(activeStepState->pid, SIGINT) == -1) printError("error killing build step ‘%s’: %s", - activeStep->step->drvPath, strerror(errno)); + localStore->printStorePath(activeStep->step->drvPath), + strerror(errno)); } } } @@ -370,8 +373,8 @@ void State::processQueueChange(Connection & conn) Step::ptr State::createStep(ref destStore, - Connection & conn, Build::ptr build, const Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable) { if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0; @@ -400,7 +403,7 @@ Step::ptr State::createStep(ref destStore, /* If it doesn't exist, create it. */ if (!step) { step = std::make_shared(); - step->drvPath = drvPath; + step->drvPath = drvPath.clone(); isNew = true; } @@ -414,28 +417,28 @@ Step::ptr State::createStep(ref destStore, if (referringStep) step_->rdeps.push_back(referringStep); - (*steps_)[drvPath] = step; + steps_->insert_or_assign(drvPath.clone(), step); } if (!isNew) return step; - printMsg(lvlDebug, format("considering derivation ‘%1%’") % drvPath); + printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath)); /* Initialize the step. Note that the step may be visible in ‘steps’ before this point, but that doesn't matter because it's not runnable yet, and other threads won't make it runnable while step->created == false. */ - step->drv = readDerivation(drvPath); - step->parsedDrv = std::make_unique(drvPath, step->drv); + step->drv = std::make_unique(readDerivation(*localStore, localStore->printStorePath(drvPath))); + step->parsedDrv = std::make_unique(drvPath.clone(), *step->drv); step->preferLocalBuild = step->parsedDrv->willBuildLocally(); - step->isDeterministic = get(step->drv.env, "isDetermistic", "0") == "1"; + step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1"; - step->systemType = step->drv.platform; + step->systemType = step->drv->platform; { - auto i = step->drv.env.find("requiredSystemFeatures"); + auto i = step->drv->env.find("requiredSystemFeatures"); StringSet features; - if (i != step->drv.env.end()) + if (i != step->drv->env.end()) features = step->requiredSystemFeatures = tokenizeString>(i->second); if (step->preferLocalBuild) features.insert("local"); @@ -451,12 +454,13 @@ Step::ptr State::createStep(ref destStore, /* Are all outputs valid? */ bool valid = true; - PathSet outputs = step->drv.outputPaths(); + auto outputs = step->drv->outputPaths(); DerivationOutputs missing; - for (auto & i : step->drv.outputs) + for (auto & i : step->drv->outputs) if (!destStore->isValidPath(i.second.path)) { valid = false; - missing[i.first] = i.second; + missing.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); } /* Try to copy the missing paths from the local store or from @@ -469,7 +473,7 @@ Step::ptr State::createStep(ref destStore, avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({i.second.path}, infos); + localStore->querySubstitutablePathInfos(singleton(i.second.path), infos); if (infos.size() == 1) avail++; } @@ -482,14 +486,18 @@ Step::ptr State::createStep(ref destStore, time_t startTime = time(0); if (localStore->isValidPath(i.second.path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", i.second.path, drvPath); + printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); else { - printInfo("substituting output ‘%1%’ of ‘%2%’", i.second.path, drvPath); + printInfo("substituting output ‘%1%’ of ‘%2%’", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); localStore->ensurePath(i.second.path); // FIXME: should copy directly from substituter to destStore. } - copyClosure(ref(localStore), destStore, {i.second.path}); + copyClosure(ref(localStore), destStore, singleton(i.second.path)); time_t stopTime = time(0); @@ -501,7 +509,10 @@ Step::ptr State::createStep(ref destStore, } } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", i.second.path, drvPath, e.what()); + printError("while copying/substituting output ‘%s’ of ‘%s’: %s", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath), + e.what()); valid = false; break; } @@ -511,15 +522,15 @@ Step::ptr State::createStep(ref destStore, // FIXME: check whether all outputs are in the binary cache. if (valid) { - finishedDrvs.insert(drvPath); + finishedDrvs.insert(drvPath.clone()); return 0; } /* No, we need to build. */ - printMsg(lvlDebug, format("creating build step ‘%1%’") % drvPath); + printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); /* Create steps for the dependencies. */ - for (auto & i : step->drv.inputDrvs) { + for (auto & i : step->drv->inputDrvs) { auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); @@ -607,7 +618,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (r.empty()) continue; BuildID id = r[0][0].as(); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index a95cdb61..de74b768 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -123,8 +123,8 @@ struct Build typedef std::weak_ptr wptr; BuildID id; - nix::Path drvPath; - std::map outputs; + nix::StorePath drvPath; + std::map outputs; std::string projectName, jobsetName, jobName; time_t timestamp; unsigned int maxSilentTime, buildTimeout; @@ -150,8 +150,8 @@ struct Step typedef std::shared_ptr ptr; typedef std::weak_ptr wptr; - nix::Path drvPath; - nix::Derivation drv; + nix::StorePath drvPath; + std::unique_ptr drv; std::unique_ptr parsedDrv; std::set requiredSystemFeatures; bool preferLocalBuild; @@ -252,7 +252,7 @@ struct Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypes.count(step->drv.platform == "builtin" ? nix::settings.thisSystem : step->drv.platform)) + if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this @@ -313,7 +313,7 @@ private: queued builds). Note that these are weak pointers. Steps are kept alive by being reachable from Builds or by being in progress. */ - typedef std::map Steps; + typedef std::map Steps; nix::Sync steps; /* Build steps that have no unbuilt dependencies. */ @@ -454,7 +454,7 @@ private: const std::string & machine); int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath); + Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); @@ -473,8 +473,8 @@ private: const nix::Derivation & drv); Step::ptr createStep(nix::ref store, - Connection & conn, Build::ptr build, const nix::Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const nix::StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable); Jobset::ptr createJobset(pqxx::work & txn, @@ -523,7 +523,7 @@ private: void dumpStatus(Connection & conn, bool log); - void addRoot(const nix::Path & storePath); + void addRoot(const nix::StorePath & storePath); public: diff --git a/src/hydra-queue-runner/token-server.hh b/src/hydra-queue-runner/token-server.hh index e00004d0..d8004f73 100644 --- a/src/hydra-queue-runner/token-server.hh +++ b/src/hydra-queue-runner/token-server.hh @@ -7,7 +7,7 @@ namespace nix { -MakeError(NoTokens, Error) +MakeError(NoTokens, Error); /* This class hands out tokens. There are only ‘maxTokens’ tokens available. Calling get(N) will return a Token object, representing diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh index 5bb5aac5..b7bed815 100644 --- a/src/libhydra/db.hh +++ b/src/libhydra/db.hh @@ -12,7 +12,7 @@ struct Connection : pqxx::connection std::string getFlags() { using namespace nix; - auto s = getEnv("HYDRA_DBI", "dbi:Pg:dbname=hydra;"); + auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;"); std::string prefix = "dbi:Pg:"; if (std::string(s, 0, prefix.size()) != prefix) throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh index a4050666..98d73d47 100644 --- a/src/libhydra/hydra-config.hh +++ b/src/libhydra/hydra-config.hh @@ -14,9 +14,9 @@ struct Config /* Read hydra.conf. */ auto hydraConfigFile = getEnv("HYDRA_CONFIG"); - if (pathExists(hydraConfigFile)) { + if (hydraConfigFile && pathExists(*hydraConfigFile)) { - for (auto line : tokenizeString(readFile(hydraConfigFile), "\n")) { + for (auto line : tokenizeString(readFile(*hydraConfigFile), "\n")) { line = trim(string(line, 0, line.find('#'))); auto eq = line.find('=');