From 7d3ba616a972d773a4b7130b69d37aa22ae20025 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2020 11:33:29 +0200 Subject: [PATCH] Fix build --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 18 ++++----- src/hydra-queue-runner/build-remote.cc | 13 ++++--- src/hydra-queue-runner/build-result.cc | 7 ++-- src/hydra-queue-runner/build-result.hh | 4 +- src/hydra-queue-runner/builder.cc | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 13 ++----- src/hydra-queue-runner/nar-extractor.cc | 4 +- src/hydra-queue-runner/queue-monitor.cc | 39 +++++++++++--------- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index ff74c23e..d9e5dfa3 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -453,29 +453,29 @@ int main(int argc, char * * argv) job["constituents"].push_back(drvPath2); } } else { - std::string drvPath = job["drvPath"]; - auto drv = readDerivation(*store, drvPath); + auto drvPath = store->parseStorePath((std::string) job["drvPath"]); + auto drv = store->readDerivation(drvPath); for (std::string jobName2 : *named) { auto job2 = state->jobs.find(jobName2); if (job2 == state->jobs.end()) throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - std::string drvPath2 = (*job2)["drvPath"]; - auto drv2 = readDerivation(*store, drvPath2); - job["constituents"].push_back(drvPath2); - drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first}; + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; } - std::string drvName(store->parseStorePath(drvPath).name()); + std::string drvName(drvPath.name()); assert(hasSuffix(drvName, drvExtension)); drvName.resize(drvName.size() - drvExtension.size()); auto h = hashDerivationModulo(*store, drv, true); auto outPath = store->makeOutputPath("out", h, drvName); drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = outPath }); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); - debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); job["drvPath"] = newDrvPath; job["outputs"]["out"] = store->printStorePath(outPath); diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index e0073484..6afe3aac 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -260,12 +260,13 @@ void State::buildRemote(ref destStore, inputs.insert(p); for (auto & input : step->drv->inputDrvs) { - Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); + auto drv2 = localStore->readDerivation(input.first); for (auto & name : input.second) { - auto i = drv2.outputs.find(name); - if (i == drv2.outputs.end()) continue; - inputs.insert(i->second.path); - basicDrv.inputSrcs.insert(i->second.path); + if (auto i = get(drv2.outputs, name)) { + auto outPath = i->path(*localStore, drv2.name); + inputs.insert(outPath); + basicDrv.inputSrcs.insert(outPath); + } } } @@ -433,7 +434,7 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - auto outputs = step->drv->outputPaths(); + auto outputs = step->drv->outputPaths(*localStore); /* Get info about each output path. */ std::map infos; diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 43af82d5..a22a7115 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -16,7 +16,7 @@ BuildOutput getBuildOutput( BuildOutput res; /* Compute the closure size. */ - auto outputs = drv.outputPaths(); + auto outputs = drv.outputPaths(*store); StorePathSet closure; for (auto & output : outputs) store->computeFSClosure(output, closure); @@ -106,10 +106,11 @@ BuildOutput getBuildOutput( if (!explicitProducts) { for (auto & output : drv.outputs) { BuildProduct product; - product.path = store->printStorePath(output.second.path); + auto outPath = output.second.path(*store, drv.name); + product.path = store->printStorePath(outPath); product.type = "nix-build"; product.subtype = output.first == "out" ? "" : output.first; - product.name = output.second.path.name(); + product.name = outPath.name(); auto file = narMembers.find(product.path); assert(file != narMembers.end()); diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index a2b9a0f9..4cbbaccb 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -12,8 +12,8 @@ struct BuildProduct nix::Path path, defaultPath; std::string type, subtype, name; bool isRegular = false; - nix::Hash sha256hash; - off_t fileSize = 0; + std::optional sha256hash; + std::optional fileSize; BuildProduct() { } }; diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 16cc56a8..c5b44344 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -274,7 +274,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & path : step->drv->outputPaths()) + for (auto & path : step->drv->outputPaths(*localStore)) addRoot(path); /* Register success in the database for all Build objects that @@ -463,7 +463,7 @@ void State::failStep( /* Remember failed paths in the database so that they won't be built again. */ if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv->outputPaths()) + for (auto & path : step->drv->outputPaths(*localStore)) txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path)); txn.commit(); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 0c7e26f5..2d616840 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -260,7 +260,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID for (auto & output : step->drv->outputs) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, output.first, localStore->printStorePath(output.second.path)); + buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name))); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -412,8 +412,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, productNr++, product.type, product.subtype, - product.isRegular ? std::make_optional(product.fileSize) : std::nullopt, - product.isRegular ? std::make_optional(product.sha256hash.to_string(Base16, false)) : std::nullopt, + product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, + product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt, product.path, product.name, product.defaultPath); @@ -441,7 +441,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, bool State::checkCachedFailure(Step::ptr step, Connection & conn) { pqxx::work txn(conn); - for (auto & path : step->drv->outputPaths()) + for (auto & path : step->drv->outputPaths(*localStore)) if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty()) return true; return false; @@ -637,12 +637,7 @@ void State::dumpStatus(Connection & conn) auto nested2 = nested.object("s3"); auto & s3Stats = s3Store->getS3Stats(); nested2.attr("put", s3Stats.put); - nested2.attr("putBytes", s3Stats.putBytes); nested2.attr("putTimeMs", s3Stats.putTimeMs); - nested2.attr("putSpeed", - s3Stats.putTimeMs - ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0); nested2.attr("get", s3Stats.get); nested2.attr("getBytes", s3Stats.getBytes); nested2.attr("getTimeMs", s3Stats.getTimeMs); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index b051cd19..8705e4ce 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -39,13 +39,13 @@ struct Extractor : ParseSink std::optional expectedSize; std::unique_ptr hashSink; - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { expectedSize = size; hashSink = std::make_unique(htSHA256); } - void receiveContents(unsigned char * data, unsigned int len) override + void receiveContents(unsigned char * data, size_t len) override { assert(expectedSize); assert(curMember); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 0a900a87..ee073b53 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -176,10 +176,10 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & output : ex.step->drv->outputs) { + for (auto & output : ex.step->drv->outputPaths(*localStore)) { auto res = txn.exec_params ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(output.second.path)); + localStore->printStorePath(output)); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -218,10 +218,10 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath)); + auto drv = localStore->readDerivation(build->drvPath); BuildOutput res = getBuildOutputCached(conn, destStore, drv); - for (auto & path : drv.outputPaths()) + for (auto & path : drv.outputPaths(*localStore)) addRoot(path); { @@ -427,7 +427,7 @@ Step::ptr State::createStep(ref destStore, ‘steps’ before this point, but that doesn't matter because it's not runnable yet, and other threads won't make it runnable while step->created == false. */ - step->drv = std::make_unique(readDerivation(*localStore, localStore->printStorePath(drvPath))); + step->drv = std::make_unique(localStore->readDerivation(drvPath)); step->parsedDrv = std::make_unique(drvPath, *step->drv); step->preferLocalBuild = step->parsedDrv->willBuildLocally(); @@ -453,10 +453,10 @@ Step::ptr State::createStep(ref destStore, /* Are all outputs valid? */ bool valid = true; - auto outputs = step->drv->outputPaths(); + auto outputs = step->drv->outputPaths(*localStore); DerivationOutputs missing; for (auto & i : step->drv->outputs) - if (!destStore->isValidPath(i.second.path)) { + if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) { valid = false; missing.insert_or_assign(i.first, i.second); } @@ -467,11 +467,12 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & i : missing) { - if (/* localStore != destStore && */ localStore->isValidPath(i.second.path)) + auto path = i.second.path(*localStore, step->drv->name); + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({i.second.path}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -480,35 +481,37 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; for (auto & i : missing) { + auto path = i.second.path(*localStore, step->drv->name); + try { time_t startTime = time(0); - if (localStore->isValidPath(i.second.path)) + if (localStore->isValidPath(path)) printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(i.second.path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); else { printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(i.second.path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(i.second.path); + localStore->ensurePath(path); // FIXME: should copy directly from substituter to destStore. } - copyClosure(ref(localStore), destStore, {i.second.path}); + copyClosure(ref(localStore), destStore, {path}); time_t stopTime = time(0); { auto mc = startDbUpdate(); pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); txn.commit(); } } catch (Error & e) { printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(i.second.path), + localStore->printStorePath(path), localStore->printStorePath(drvPath), e.what()); valid = false; @@ -614,12 +617,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref { pqxx::work txn(conn); - for (auto & output : drv.outputs) { + for (auto & output : drv.outputPaths(*localStore)) { auto r = txn.exec_params ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", - localStore->printStorePath(output.second.path)); + localStore->printStorePath(output)); if (r.empty()) continue; BuildID id = r[0][0].as();