From 5b4df3ad5a2b318a29b6bebc3e6be353b8c61d65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Jul 2020 20:38:59 +0200 Subject: [PATCH] Get data needed by getBuildOutput() from the incoming NAR in a streaming fashion --- src/hydra-queue-runner/Makefile.am | 5 +- src/hydra-queue-runner/build-remote.cc | 17 +++-- src/hydra-queue-runner/build-result.cc | 67 ++++++++++++-------- src/hydra-queue-runner/build-result.hh | 7 ++- src/hydra-queue-runner/builder.cc | 8 +-- src/hydra-queue-runner/nar-extractor.cc | 82 +++++++++++++++++++++++++ src/hydra-queue-runner/nar-extractor.hh | 23 +++++++ src/hydra-queue-runner/queue-monitor.cc | 3 +- src/hydra-queue-runner/state.hh | 5 +- 9 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 src/hydra-queue-runner/nar-extractor.cc create mode 100644 src/hydra-queue-runner/nar-extractor.hh diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 9582a629..ea852334 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -1,7 +1,8 @@ bin_PROGRAMS = hydra-queue-runner hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ - builder.cc build-result.cc build-remote.cc \ - build-result.hh counter.hh state.hh db.hh + builder.cc build-result.cc build-remote.cc \ + build-result.hh counter.hh state.hh db.hh \ + nar-extractor.cc nar-extractor.hh hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 26d991ef..b7d5ff85 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -160,7 +160,8 @@ void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, RemoteResult & result, std::shared_ptr activeStep, - std::function updateStep) + std::function updateStep, + NarMemberDatas & narMembers) { assert(BuildResult::TimedOut == 8); @@ -427,8 +428,6 @@ void State::buildRemote(ref destStore, } /* Copy the output paths. */ - result.accessor = destStore->getFSAccessor(); - if (!machine->isLocalhost() || localStore != std::shared_ptr(destStore)) { updateStep(ssReceivingOutputs); @@ -475,7 +474,17 @@ void State::buildRemote(ref destStore, auto & info = infos.find(path)->second; to << cmdDumpStorePath << localStore->printStorePath(path); to.flush(); - destStore->addToStore(info, from); + + /* Receive the NAR from the remote and add it to the + destination store. Meanwhile, extract all the info from the + NAR that getBuildOutput() needs. */ + auto source2 = sinkToSource([&](Sink & sink) + { + TeeSource tee(from, sink); + extractNarData(tee, localStore->printStorePath(path), narMembers); + }); + + destStore->addToStore(info, *source2); } auto now2 = std::chrono::steady_clock::now(); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 868ccb15..43af82d5 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -8,8 +8,10 @@ using namespace nix; -BuildOutput getBuildOutput(nix::ref store, - nix::ref accessor, const Derivation & drv) +BuildOutput getBuildOutput( + nix::ref store, + NarMemberDatas & narMembers, + const Derivation & drv) { BuildOutput res; @@ -24,6 +26,20 @@ BuildOutput getBuildOutput(nix::ref store, if (outputs.count(path)) res.size += info->narSize; } + /* Fetch missing data. Usually buildRemote() will have extracted + this data from the incoming NARs. */ + for (auto & output : outputs) { + auto outputS = store->printStorePath(output); + if (!narMembers.count(outputS)) { + printInfo("fetching NAR contents of '%s'...", outputS); + auto source = sinkToSource([&](Sink & sink) + { + store->narFromPath(output, sink); + }); + extractNarData(*source, outputS, narMembers); + } + } + /* Get build products. */ bool explicitProducts = false; @@ -39,17 +55,18 @@ BuildOutput getBuildOutput(nix::ref store, for (auto & output : outputs) { auto outputS = store->printStorePath(output); - Path failedFile = outputS + "/nix-support/failed"; - if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) + if (narMembers.count(outputS + "/nix-support/failed")) res.failed = true; - Path productsFile = outputS + "/nix-support/hydra-build-products"; - if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) + auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products"); + if (productsFile == narMembers.end() || + productsFile->second.type != FSAccessor::Type::tRegular) continue; + assert(productsFile->second.contents); explicitProducts = true; - for (auto & line : tokenizeString(accessor->readFile(productsFile), "\n")) { + for (auto & line : tokenizeString(productsFile->second.contents.value(), "\n")) { BuildProduct product; std::smatch match; @@ -69,16 +86,15 @@ BuildOutput getBuildOutput(nix::ref store, product.path = canonPath(product.path); if (!store->isInStore(product.path)) continue; - auto st = accessor->stat(product.path); - if (st.type == FSAccessor::Type::tMissing) continue; + auto file = narMembers.find(product.path); + if (file == narMembers.end()) continue; product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); - if (st.type == FSAccessor::Type::tRegular) { + if (file->second.type == FSAccessor::Type::tRegular) { product.isRegular = true; - product.fileSize = st.fileSize; - auto contents = accessor->readFile(product.path); - product.sha256hash = hashString(htSHA256, contents); + product.fileSize = file->second.fileSize.value(); + product.sha256hash = file->second.sha256.value(); } res.products.push_back(product); @@ -95,29 +111,30 @@ BuildOutput getBuildOutput(nix::ref store, product.subtype = output.first == "out" ? "" : output.first; product.name = output.second.path.name(); - auto st = accessor->stat(product.path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("getting status of ā€˜%sā€™", product.path); - if (st.type == FSAccessor::Type::tDirectory) + auto file = narMembers.find(product.path); + assert(file != narMembers.end()); + if (file->second.type == FSAccessor::Type::tDirectory) res.products.push_back(product); } } /* Get the release name from $output/nix-support/hydra-release-name. */ for (auto & output : outputs) { - auto p = store->printStorePath(output) + "/nix-support/hydra-release-name"; - if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; - try { - res.releaseName = trim(accessor->readFile(p)); - } catch (Error & e) { continue; } + auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name"); + if (file == narMembers.end() || + file->second.type != FSAccessor::Type::tRegular) + continue; + res.releaseName = trim(file->second.contents.value()); // FIXME: validate release name } /* Get metrics. */ for (auto & output : outputs) { - auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics"; - if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; - for (auto & line : tokenizeString(accessor->readFile(metricsFile), "\n")) { + auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics"); + if (file == narMembers.end() || + file->second.type != FSAccessor::Type::tRegular) + continue; + for (auto & line : tokenizeString(file->second.contents.value(), "\n")) { auto fields = tokenizeString>(line); if (fields.size() < 2) continue; BuildMetric metric; diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index 57676797..a2b9a0f9 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "derivations.hh" #include "store-api.hh" +#include "nar-extractor.hh" struct BuildProduct { @@ -38,5 +39,7 @@ struct BuildOutput std::map metrics; }; -BuildOutput getBuildOutput(nix::ref store, - nix::ref accessor, const nix::Derivation & drv); +BuildOutput getBuildOutput( + nix::ref store, + NarMemberDatas & narMembers, + const nix::Derivation & drv); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index bd1e64a7..16cc56a8 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -201,9 +201,11 @@ State::StepResult State::doBuildStep(nix::ref destStore, }; /* Do the build. */ + NarMemberDatas narMembers; + try { /* FIXME: referring builds may have conflicting timeouts. */ - buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep); + buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers); } catch (Error & e) { if (activeStep->state_.lock()->cancelled) { printInfo("marking step %d of build %d as cancelled", stepNr, buildId); @@ -218,10 +220,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, ref(result.accessor), *step->drv); + res = getBuildOutput(destStore, narMembers, *step->drv); } - - result.accessor = 0; } time_t stepStopTime = time(0); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc new file mode 100644 index 00000000..b051cd19 --- /dev/null +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -0,0 +1,82 @@ +#include "nar-extractor.hh" + +#include "archive.hh" + +#include + +using namespace nix; + +struct Extractor : ParseSink +{ + std::unordered_set filesToKeep { + "/nix-support/hydra-build-products", + "/nix-support/hydra-release-name", + "/nix-support/hydra-metrics", + }; + + NarMemberDatas & members; + NarMemberData * curMember = nullptr; + Path prefix; + + Extractor(NarMemberDatas & members, const Path & prefix) + : members(members), prefix(prefix) + { } + + void createDirectory(const Path & path) override + { + members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory }); + } + + void createRegularFile(const Path & path) override + { + curMember = &members.insert_or_assign(prefix + path, NarMemberData { + .type = FSAccessor::Type::tRegular, + .fileSize = 0, + .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, + }).first->second; + } + + std::optional expectedSize; + std::unique_ptr hashSink; + + void preallocateContents(unsigned long long size) override + { + expectedSize = size; + hashSink = std::make_unique(htSHA256); + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + assert(expectedSize); + assert(curMember); + assert(hashSink); + *curMember->fileSize += len; + (*hashSink)(data, len); + if (curMember->contents) { + curMember->contents->append((char *) data, len); + } + assert(curMember->fileSize <= expectedSize); + if (curMember->fileSize == expectedSize) { + auto [hash, len] = hashSink->finish(); + assert(curMember->fileSize == len); + curMember->sha256 = hash; + hashSink.reset(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink }); + } +}; + + +void extractNarData( + Source & source, + const Path & prefix, + NarMemberDatas & members) +{ + Extractor extractor(members, prefix); + parseDump(extractor, source); + // Note: this point may not be reached if we're in a coroutine. +} diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh new file mode 100644 index 00000000..d14726a3 --- /dev/null +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "fs-accessor.hh" +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" + +struct NarMemberData +{ + nix::FSAccessor::Type type; + std::optional fileSize; + std::optional contents; + std::optional sha256; +}; + +typedef std::map NarMemberDatas; + +/* Read a NAR from a source and get to some info about every file + inside the NAR. */ +void extractNarData( + nix::Source & source, + const nix::Path & prefix, + NarMemberDatas & members); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 5bfcb1eb..0a900a87 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -672,5 +672,6 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref } - return getBuildOutput(destStore, destStore->getFSAccessor(), drv); + NarMemberDatas narMembers; + return getBuildOutput(destStore, narMembers, drv); } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 28364db0..57fd5a77 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -14,6 +14,7 @@ #include "pool.hh" #include "store-api.hh" #include "sync.hh" +#include "nar-extractor.hh" typedef unsigned int BuildID; @@ -64,7 +65,6 @@ struct RemoteResult time_t startTime = 0, stopTime = 0; unsigned int overhead = 0; nix::Path logFile; - std::shared_ptr accessor; BuildStatus buildStatus() const { @@ -518,7 +518,8 @@ private: unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, RemoteResult & result, std::shared_ptr activeStep, - std::function updateStep); + std::function updateStep, + NarMemberDatas & narMembers); void markSucceededBuild(pqxx::work & txn, Build::ptr build, const BuildOutput & res, bool isCachedBuild, time_t startTime, time_t stopTime);