snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Fix build

Browse source
This commit is contained in:
Eelco Dolstra 2020-06-23 13:43:54 +02:00
parent 1726adbad7
commit bb32aafa4a
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
11 changed files with 39 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/hydra-eval-jobs/Makefile.am
View file

@ -1,5 +1,5 @@
bin_PROGRAMS = hydra-eval-jobs bin_PROGRAMS = hydra-eval-jobs
hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust hydra_eval_jobs_LDADD = $(NIX_LIBS)
hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra

6
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -472,7 +472,11 @@ int main(int argc, char * * argv)
auto h = hashDerivationModulo(*store, drv, true); auto h = hashDerivationModulo(*store, drv, true);
auto outPath = store->makeOutputPath("out", h, drvName); auto outPath = store->makeOutputPath("out", h, drvName);
drv.env["out"] = store->printStorePath(outPath); drv.env["out"] = store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", "")); drv.outputs.insert_or_assign("out", DerivationOutput {
.path = outPath,
.hashAlgo = "",
.hash = ""
});
auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath);

2
src/hydra-evaluator/hydra-evaluator.cc
View file

@ -140,7 +140,7 @@ struct Evaluator
jobset.pid = startProcess([&]() { jobset.pid = startProcess([&]() {
Strings args = { "hydra-eval-jobset", jobset.name.first, jobset.name.second }; Strings args = { "hydra-eval-jobset", jobset.name.first, jobset.name.second };
execvp(args.front().c_str(), stringsToCharPtrs(args).data()); execvp(args.front().c_str(), stringsToCharPtrs(args).data());
throw SysError(format("executing ‘%1%’") % args.front()); throw SysError("executing ‘%1%’", args.front());
}); });
state.runningEvals++; state.runningEvals++;

2
src/hydra-queue-runner/Makefile.am
View file

@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \ builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh token-server.hh state.hh db.hh build-result.hh counter.hh token-server.hh state.hh db.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

26
src/hydra-queue-runner/build-remote.cc
View file

@ -108,9 +108,9 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
StorePathSet missing; StorePathSet missing;
for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
if (!present.count(*i)) missing.insert(i->clone()); if (!present.count(*i)) missing.insert(*i);
printMsg(lvlDebug, format("sending %1% missing paths") % missing.size()); printMsg(lvlDebug, "sending %d missing paths", missing.size());
std::unique_lock<std::timed_mutex> sendLock(sendMutex, std::unique_lock<std::timed_mutex> sendLock(sendMutex,
std::chrono::seconds(600)); std::chrono::seconds(600));
@ -139,7 +139,7 @@ void State::buildRemote(ref<Store> destStore,
createDirs(dirOf(result.logFile)); createDirs(dirOf(result.logFile));
AutoCloseFD logFD = open(result.logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); AutoCloseFD logFD = open(result.logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666);
if (!logFD) throw SysError(format("creating log file ‘%1%’") % result.logFile); if (!logFD) throw SysError("creating log file ‘%s’", result.logFile);
nix::Path tmpDir = createTempDir(); nix::Path tmpDir = createTempDir();
AutoDelete tmpDirDel(tmpDir, true); AutoDelete tmpDirDel(tmpDir, true);
@ -187,10 +187,10 @@ void State::buildRemote(ref<Store> destStore,
unsigned int magic = readInt(from); unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_2) if (magic != SERVE_MAGIC_2)
throw Error(format("protocol mismatch with ‘nix-store --serve’ on ‘%1%’") % machine->sshName); throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", machine->sshName);
remoteVersion = readInt(from); remoteVersion = readInt(from);
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
throw Error(format("unsupported ‘nix-store --serve’ protocol version on ‘%1%’") % machine->sshName); throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", machine->sshName);
// Always send the derivation to localhost, since it's a // Always send the derivation to localhost, since it's a
// no-op anyway but we might not be privileged to use // no-op anyway but we might not be privileged to use
// cmdBuildDerivation (e.g. if we're running in a NixOS // cmdBuildDerivation (e.g. if we're running in a NixOS
@ -203,7 +203,7 @@ void State::buildRemote(ref<Store> destStore,
} catch (EndOfFile & e) { } catch (EndOfFile & e) {
child.pid.wait(); child.pid.wait();
string s = chomp(readFile(result.logFile)); string s = chomp(readFile(result.logFile));
throw Error(format("cannot connect to ‘%1%’: %2%") % machine->sshName % s); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
} }
{ {
@ -222,18 +222,18 @@ void State::buildRemote(ref<Store> destStore,
BasicDerivation basicDrv(*step->drv); BasicDerivation basicDrv(*step->drv);
if (sendDerivation) if (sendDerivation)
inputs.insert(step->drvPath.clone()); inputs.insert(step->drvPath);
else else
for (auto & p : step->drv->inputSrcs) for (auto & p : step->drv->inputSrcs)
inputs.insert(p.clone()); inputs.insert(p);
for (auto & input : step->drv->inputDrvs) { for (auto & input : step->drv->inputDrvs) {
Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
for (auto & name : input.second) { for (auto & name : input.second) {
auto i = drv2.outputs.find(name); auto i = drv2.outputs.find(name);
if (i == drv2.outputs.end()) continue; if (i == drv2.outputs.end()) continue;
inputs.insert(i->second.path.clone()); inputs.insert(i->second.path);
basicDrv.inputSrcs.insert(i->second.path.clone()); basicDrv.inputSrcs.insert(i->second.path);
} }
} }
@ -282,7 +282,7 @@ void State::buildRemote(ref<Store> destStore,
if (sendDerivation) { if (sendDerivation) {
to << cmdBuildPaths; to << cmdBuildPaths;
writeStorePaths(*localStore, to, singleton(step->drvPath)); writeStorePaths(*localStore, to, {step->drvPath});
} else { } else {
to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
writeDerivation(to, *localStore, basicDrv); writeDerivation(to, *localStore, basicDrv);
@ -306,7 +306,7 @@ void State::buildRemote(ref<Store> destStore,
if (sendDerivation) { if (sendDerivation) {
if (res) { if (res) {
result.errorMsg = (format("%1% on ‘%2%’") % readString(from) % machine->sshName).str(); result.errorMsg = fmt("%s on ‘%s’", readString(from), machine->sshName);
if (res == 100) { if (res == 100) {
result.stepStatus = bsFailed; result.stepStatus = bsFailed;
result.canCache = true; result.canCache = true;
@ -472,7 +472,7 @@ void State::buildRemote(ref<Store> destStore,
info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4); info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
info->lastFailure = now; info->lastFailure = now;
int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30); int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
printMsg(lvlInfo, format("will disable machine ‘%1%’ for %2%s") % machine->sshName % delta); printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
info->disabledUntil = now + std::chrono::seconds(delta); info->disabledUntil = now + std::chrono::seconds(delta);
} }
throw; throw;

2
src/hydra-queue-runner/build-result.cc
View file

@ -17,7 +17,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
auto outputs = drv.outputPaths(); auto outputs = drv.outputPaths();
StorePathSet closure; StorePathSet closure;
for (auto & output : outputs) for (auto & output : outputs)
store->computeFSClosure(singleton(output), closure); store->computeFSClosure(output, closure);
for (auto & path : closure) { for (auto & path : closure) {
auto info = store->queryPathInfo(path); auto info = store->queryPathInfo(path);
res.closureSize += info->narSize; res.closureSize += info->narSize;

2
src/hydra-queue-runner/builder.cc
View file

@ -140,7 +140,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
if (!build) build = *dependents.begin(); if (!build) build = *dependents.begin();
buildId = build->id; buildId = build->id;
buildDrvPath = build->drvPath.clone(); buildDrvPath = build->drvPath;
maxSilentTime = build->maxSilentTime; maxSilentTime = build->maxSilentTime;
buildTimeout = build->buildTimeout; buildTimeout = build->buildTimeout;

12
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -187,7 +187,7 @@ void State::monitorMachinesFile()
struct stat st; struct stat st;
if (stat(machinesFile.c_str(), &st) != 0) { if (stat(machinesFile.c_str(), &st) != 0) {
if (errno != ENOENT) if (errno != ENOENT)
throw SysError(format("getting stats about ‘%1%’") % machinesFile); throw SysError("getting stats about ‘%s’", machinesFile);
st.st_ino = st.st_mtime = 0; st.st_ino = st.st_mtime = 0;
} }
auto & old(fileStats[n]); auto & old(fileStats[n]);
@ -219,7 +219,7 @@ void State::monitorMachinesFile()
// FIXME: use inotify. // FIXME: use inotify.
sleep(30); sleep(30);
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("reloading machines file: %1%") % e.what()); printMsg(lvlError, "reloading machines file: %s", e.what());
sleep(5); sleep(5);
} }
} }
@ -804,7 +804,7 @@ void State::run(BuildID buildOne)
auto conn(dbPool.get()); auto conn(dbPool.get());
pqxx::work txn(*conn); pqxx::work txn(*conn);
for (auto & step : steps) { for (auto & step : steps) {
printMsg(lvlError, format("cleaning orphaned step %d of build %d") % step.second % step.first); printMsg(lvlError, "cleaning orphaned step %d of build %d", step.second, step.first);
txn.exec_params0 txn.exec_params0
("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy != 0", ("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy != 0",
(int) bsAborted, (int) bsAborted,
@ -813,7 +813,7 @@ void State::run(BuildID buildOne)
} }
txn.commit(); txn.commit();
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("cleanup thread: %1%") % e.what()); printMsg(lvlError, "cleanup thread: %s", e.what());
auto orphanedSteps_(orphanedSteps.lock()); auto orphanedSteps_(orphanedSteps.lock());
orphanedSteps_->insert(steps.begin(), steps.end()); orphanedSteps_->insert(steps.begin(), steps.end());
} }
@ -829,7 +829,7 @@ void State::run(BuildID buildOne)
if (auto remoteStore = getDestStore().dynamic_pointer_cast<RemoteStore>()) if (auto remoteStore = getDestStore().dynamic_pointer_cast<RemoteStore>())
remoteStore->flushBadConnections(); remoteStore->flushBadConnections();
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("connection flush thread: %1%") % e.what()); printMsg(lvlError, "connection flush thread: %s", e.what());
} }
} }
}).detach(); }).detach();
@ -845,7 +845,7 @@ void State::run(BuildID buildOne)
dumpStatus(*conn); dumpStatus(*conn);
} }
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("main thread: %1%") % e.what()); printMsg(lvlError, "main thread: %s", e.what());
sleep(10); // probably a DB problem, so don't retry right away sleep(10); // probably a DB problem, so don't retry right away
} }
} }

16
src/hydra-queue-runner/queue-monitor.cc
View file

@ -117,7 +117,7 @@ bool State::getQueuedBuilds(Connection & conn,
newIDs.push_back(id); newIDs.push_back(id);
newBuildsByID[id] = build; newBuildsByID[id] = build;
newBuildsByPath.emplace(std::make_pair(build->drvPath.clone(), id)); newBuildsByPath.emplace(std::make_pair(build->drvPath, id));
} }
} }
@ -402,7 +402,7 @@ Step::ptr State::createStep(ref<Store> destStore,
/* If it doesn't exist, create it. */ /* If it doesn't exist, create it. */
if (!step) { if (!step) {
step = std::make_shared<Step>(drvPath.clone()); step = std::make_shared<Step>(drvPath);
isNew = true; isNew = true;
} }
@ -416,7 +416,7 @@ Step::ptr State::createStep(ref<Store> destStore,
if (referringStep) if (referringStep)
step_->rdeps.push_back(referringStep); step_->rdeps.push_back(referringStep);
steps_->insert_or_assign(drvPath.clone(), step); steps_->insert_or_assign(drvPath, step);
} }
if (!isNew) return step; if (!isNew) return step;
@ -428,7 +428,7 @@ Step::ptr State::createStep(ref<Store> destStore,
it's not runnable yet, and other threads won't make it it's not runnable yet, and other threads won't make it
runnable while step->created == false. */ runnable while step->created == false. */
step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath))); step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath)));
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *step->drv); step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
step->preferLocalBuild = step->parsedDrv->willBuildLocally(); step->preferLocalBuild = step->parsedDrv->willBuildLocally();
step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1"; step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1";
@ -459,7 +459,7 @@ Step::ptr State::createStep(ref<Store> destStore,
if (!destStore->isValidPath(i.second.path)) { if (!destStore->isValidPath(i.second.path)) {
valid = false; valid = false;
missing.insert_or_assign(i.first, missing.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); DerivationOutput { i.second.path, i.second.hashAlgo, i.second.hash });
} }
/* Try to copy the missing paths from the local store or from /* Try to copy the missing paths from the local store or from
@ -472,7 +472,7 @@ Step::ptr State::createStep(ref<Store> destStore,
avail++; avail++;
else if (useSubstitutes) { else if (useSubstitutes) {
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos(singleton(i.second.path), infos); localStore->querySubstitutablePathInfos({i.second.path}, infos);
if (infos.size() == 1) if (infos.size() == 1)
avail++; avail++;
} }
@ -496,7 +496,7 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
copyClosure(ref<Store>(localStore), destStore, singleton(i.second.path)); copyClosure(ref<Store>(localStore), destStore, {i.second.path});
time_t stopTime = time(0); time_t stopTime = time(0);
@ -521,7 +521,7 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: check whether all outputs are in the binary cache. // FIXME: check whether all outputs are in the binary cache.
if (valid) { if (valid) {
finishedDrvs.insert(drvPath.clone()); finishedDrvs.insert(drvPath);
return 0; return 0;
} }

2
src/hydra-queue-runner/state.hh
View file

@ -208,7 +208,7 @@ struct Step
nix::Sync<State> state; nix::Sync<State> state;
Step(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) Step(const nix::StorePath & drvPath) : drvPath(drvPath)
{ } { }
~Step() ~Step()

2
src/hydra-queue-runner/token-server.hh
View file

@ -39,7 +39,7 @@ public:
: ts(ts), tokens(tokens) : ts(ts), tokens(tokens)
{ {
if (tokens >= ts->maxTokens) if (tokens >= ts->maxTokens)
throw NoTokens(format("requesting more tokens (%d) than exist (%d)") % tokens % ts->maxTokens); throw NoTokens("requesting more tokens (%d) than exist (%d)", tokens, ts->maxTokens);
debug("acquiring %d tokens", tokens); debug("acquiring %d tokens", tokens);
auto inUse(ts->inUse.lock()); auto inUse(ts->inUse.lock());
while (*inUse + tokens > ts->maxTokens) while (*inUse + tokens > ts->maxTokens)