Get data needed by getBuildOutput() from the incoming NAR in a streaming fashion

This commit is contained in:
Eelco Dolstra 2020-07-27 20:38:59 +02:00
parent d4e4be4fd1
commit 5b4df3ad5a
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
9 changed files with 177 additions and 40 deletions

View file

@ -2,6 +2,7 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh
build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

View file

@ -160,7 +160,8 @@ void State::buildRemote(ref<Store> destStore,
Machine::ptr machine, Step::ptr step,
unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
std::function<void(StepState)> updateStep)
std::function<void(StepState)> updateStep,
NarMemberDatas & narMembers)
{
assert(BuildResult::TimedOut == 8);
@ -427,8 +428,6 @@ void State::buildRemote(ref<Store> destStore,
}
/* Copy the output paths. */
result.accessor = destStore->getFSAccessor();
if (!machine->isLocalhost() || localStore != std::shared_ptr<Store>(destStore)) {
updateStep(ssReceivingOutputs);
@ -475,7 +474,17 @@ void State::buildRemote(ref<Store> destStore,
auto & info = infos.find(path)->second;
to << cmdDumpStorePath << localStore->printStorePath(path);
to.flush();
destStore->addToStore(info, from);
/* Receive the NAR from the remote and add it to the
destination store. Meanwhile, extract all the info from the
NAR that getBuildOutput() needs. */
auto source2 = sinkToSource([&](Sink & sink)
{
TeeSource tee(from, sink);
extractNarData(tee, localStore->printStorePath(path), narMembers);
});
destStore->addToStore(info, *source2);
}
auto now2 = std::chrono::steady_clock::now();

View file

@ -8,8 +8,10 @@
using namespace nix;
BuildOutput getBuildOutput(nix::ref<Store> store,
nix::ref<nix::FSAccessor> accessor, const Derivation & drv)
BuildOutput getBuildOutput(
nix::ref<Store> store,
NarMemberDatas & narMembers,
const Derivation & drv)
{
BuildOutput res;
@ -24,6 +26,20 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
if (outputs.count(path)) res.size += info->narSize;
}
/* Fetch missing data. Usually buildRemote() will have extracted
this data from the incoming NARs. */
for (auto & output : outputs) {
auto outputS = store->printStorePath(output);
if (!narMembers.count(outputS)) {
printInfo("fetching NAR contents of '%s'...", outputS);
auto source = sinkToSource([&](Sink & sink)
{
store->narFromPath(output, sink);
});
extractNarData(*source, outputS, narMembers);
}
}
/* Get build products. */
bool explicitProducts = false;
@ -39,17 +55,18 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
for (auto & output : outputs) {
auto outputS = store->printStorePath(output);
Path failedFile = outputS + "/nix-support/failed";
if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)
if (narMembers.count(outputS + "/nix-support/failed"))
res.failed = true;
Path productsFile = outputS + "/nix-support/hydra-build-products";
if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)
auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
if (productsFile == narMembers.end() ||
productsFile->second.type != FSAccessor::Type::tRegular)
continue;
assert(productsFile->second.contents);
explicitProducts = true;
for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {
for (auto & line : tokenizeString<Strings>(productsFile->second.contents.value(), "\n")) {
BuildProduct product;
std::smatch match;
@ -69,16 +86,15 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
product.path = canonPath(product.path);
if (!store->isInStore(product.path)) continue;
auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing) continue;
auto file = narMembers.find(product.path);
if (file == narMembers.end()) continue;
product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
if (st.type == FSAccessor::Type::tRegular) {
if (file->second.type == FSAccessor::Type::tRegular) {
product.isRegular = true;
product.fileSize = st.fileSize;
auto contents = accessor->readFile(product.path);
product.sha256hash = hashString(htSHA256, contents);
product.fileSize = file->second.fileSize.value();
product.sha256hash = file->second.sha256.value();
}
res.products.push_back(product);
@ -95,29 +111,30 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
product.subtype = output.first == "out" ? "" : output.first;
product.name = output.second.path.name();
auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing)
throw Error("getting status of %s", product.path);
if (st.type == FSAccessor::Type::tDirectory)
auto file = narMembers.find(product.path);
assert(file != narMembers.end());
if (file->second.type == FSAccessor::Type::tDirectory)
res.products.push_back(product);
}
}
/* Get the release name from $output/nix-support/hydra-release-name. */
for (auto & output : outputs) {
auto p = store->printStorePath(output) + "/nix-support/hydra-release-name";
if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
try {
res.releaseName = trim(accessor->readFile(p));
} catch (Error & e) { continue; }
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
continue;
res.releaseName = trim(file->second.contents.value());
// FIXME: validate release name
}
/* Get metrics. */
for (auto & output : outputs) {
auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics";
if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
continue;
for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line);
if (fields.size() < 2) continue;
BuildMetric metric;

View file

@ -5,6 +5,7 @@
#include "hash.hh"
#include "derivations.hh"
#include "store-api.hh"
#include "nar-extractor.hh"
struct BuildProduct
{
@ -38,5 +39,7 @@ struct BuildOutput
std::map<std::string, BuildMetric> metrics;
};
BuildOutput getBuildOutput(nix::ref<nix::Store> store,
nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);
BuildOutput getBuildOutput(
nix::ref<nix::Store> store,
NarMemberDatas & narMembers,
const nix::Derivation & drv);

View file

@ -201,9 +201,11 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
};
/* Do the build. */
NarMemberDatas narMembers;
try {
/* FIXME: referring builds may have conflicting timeouts. */
buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep);
buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers);
} catch (Error & e) {
if (activeStep->state_.lock()->cancelled) {
printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
@ -218,10 +220,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
if (result.stepStatus == bsSuccess) {
updateStep(ssPostProcessing);
res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), *step->drv);
res = getBuildOutput(destStore, narMembers, *step->drv);
}
result.accessor = 0;
}
time_t stepStopTime = time(0);

View file

@ -0,0 +1,82 @@
#include "nar-extractor.hh"
#include "archive.hh"
#include <unordered_set>
using namespace nix;
struct Extractor : ParseSink
{
std::unordered_set<Path> filesToKeep {
"/nix-support/hydra-build-products",
"/nix-support/hydra-release-name",
"/nix-support/hydra-metrics",
};
NarMemberDatas & members;
NarMemberData * curMember = nullptr;
Path prefix;
Extractor(NarMemberDatas & members, const Path & prefix)
: members(members), prefix(prefix)
{ }
void createDirectory(const Path & path) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
}
void createRegularFile(const Path & path) override
{
curMember = &members.insert_or_assign(prefix + path, NarMemberData {
.type = FSAccessor::Type::tRegular,
.fileSize = 0,
.contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
}).first->second;
}
std::optional<unsigned long long> expectedSize;
std::unique_ptr<HashSink> hashSink;
void preallocateContents(unsigned long long size) override
{
expectedSize = size;
hashSink = std::make_unique<HashSink>(htSHA256);
}
void receiveContents(unsigned char * data, unsigned int len) override
{
assert(expectedSize);
assert(curMember);
assert(hashSink);
*curMember->fileSize += len;
(*hashSink)(data, len);
if (curMember->contents) {
curMember->contents->append((char *) data, len);
}
assert(curMember->fileSize <= expectedSize);
if (curMember->fileSize == expectedSize) {
auto [hash, len] = hashSink->finish();
assert(curMember->fileSize == len);
curMember->sha256 = hash;
hashSink.reset();
}
}
void createSymlink(const Path & path, const string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
}
};
void extractNarData(
Source & source,
const Path & prefix,
NarMemberDatas & members)
{
Extractor extractor(members, prefix);
parseDump(extractor, source);
// Note: this point may not be reached if we're in a coroutine.
}

View file

@ -0,0 +1,23 @@
#pragma once
#include "fs-accessor.hh"
#include "types.hh"
#include "serialise.hh"
#include "hash.hh"
struct NarMemberData
{
nix::FSAccessor::Type type;
std::optional<unsigned long long> fileSize;
std::optional<std::string> contents;
std::optional<nix::Hash> sha256;
};
typedef std::map<nix::Path, NarMemberData> NarMemberDatas;
/* Read a NAR from a source and get to some info about every file
inside the NAR. */
void extractNarData(
nix::Source & source,
const nix::Path & prefix,
NarMemberDatas & members);

View file

@ -672,5 +672,6 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
}
return getBuildOutput(destStore, destStore->getFSAccessor(), drv);
NarMemberDatas narMembers;
return getBuildOutput(destStore, narMembers, drv);
}

View file

@ -14,6 +14,7 @@
#include "pool.hh"
#include "store-api.hh"
#include "sync.hh"
#include "nar-extractor.hh"
typedef unsigned int BuildID;
@ -64,7 +65,6 @@ struct RemoteResult
time_t startTime = 0, stopTime = 0;
unsigned int overhead = 0;
nix::Path logFile;
std::shared_ptr<nix::FSAccessor> accessor;
BuildStatus buildStatus() const
{
@ -518,7 +518,8 @@ private:
unsigned int maxSilentTime, unsigned int buildTimeout,
unsigned int repeats,
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
std::function<void(StepState)> updateStep);
std::function<void(StepState)> updateStep,
NarMemberDatas & narMembers);
void markSucceededBuild(pqxx::work & txn, Build::ptr build,
const BuildOutput & res, bool isCachedBuild, time_t startTime, time_t stopTime);