Store metadata about drv outputs realisations

For each known realisation, store:
- its output
- its output path

This comes with a set of needed changes:

- New `realisations` module declaring the types needed for describing
  these mappings
- New `Store::registerDrvOutput` method registering all the needed informations
  about a derivation output (also replaces `LocalStore::linkDeriverToPath`)
- new `Store::queryRealisation` method to retrieve the informations for a
  derivations

This introcudes some redundancy on the remote-store side between
`wopQueryDerivationOutputMap` and `wopQueryRealisation`.
However we might need to keep both (regardless of backwards compat)
because we sometimes need to get some infos for all the outputs of a
derivation (where `wopQueryDerivationOutputMap` is handy), but all the
stores can't implement it − because listing all the outputs of a
derivation isn't really possible for binary caches where the server
doesn't allow to list a directory.
This commit is contained in:
regnat 2020-10-08 17:36:51 +02:00
parent 9c143c411b
commit 58cdab64ac
16 changed files with 248 additions and 12 deletions

View file

@ -443,6 +443,23 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
})->path; })->path;
} }
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
{
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) };
} else {
return std::nullopt;
}
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath");
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor() ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{ {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);

View file

@ -33,6 +33,9 @@ private:
protected: protected:
// The prefix under which realisation infos will be stored
const std::string realisationsPrefix = "/realisations";
BinaryCacheStore(const Params & params); BinaryCacheStore(const Params & params);
public: public:
@ -99,6 +102,10 @@ public:
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

View file

@ -2094,6 +2094,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
/* Nothing to be done; 'path' must already be valid. */ /* Nothing to be done; 'path' must already be valid. */
} }
void registerDrvOutput(const Realisation & info) override
{
if (!goal.isAllowed(info.id.drvPath))
throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath));
next->registerDrvOutput(info);
}
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
{
if (!goal.isAllowed(id.drvPath))
throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath));
return next->queryRealisation(id);
}
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
{ {
if (buildMode != bmNormal) throw Error("unsupported build mode"); if (buildMode != bmNormal) throw Error("unsupported build mode");
@ -3393,7 +3407,10 @@ void DerivationGoal::registerOutputs()
if (useDerivation || isCaFloating) if (useDerivation || isCaFloating)
for (auto & [outputName, newInfo] : infos) for (auto & [outputName, newInfo] : infos)
worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); worker.store.registerDrvOutput(
DrvOutputId{drvPathResolved, outputName},
DrvOutputInfo{.outPath = newInfo.path,
.resolvedDrv = drvPathResolved});
} }

View file

@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopRegisterDrvOutput: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
auto resolvedDrv = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
logger->stopWork();
break;
}
case wopQueryRealisation: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId);
logger->stopWork();
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
break;
}
default: default:
throw Error("invalid operation %1%", op); throw Error("invalid operation %1%", op);
} }

View file

@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override BuildMode buildMode) override
{ unsupported("buildDerivation"); } { unsupported("buildDerivation"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
}; };
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

View file

@ -333,6 +333,10 @@ public:
auto conn(connections->get()); auto conn(connections->get());
return conn->remoteVersion; return conn->remoteVersion;
} }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
// TODO: Implement
{ unsupported("queryRealisation"); }
}; };
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore; static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;

View file

@ -87,6 +87,7 @@ protected:
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init()
{ {
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + realisationsPrefix);
if (writeDebugInfo) if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo"); createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init(); BinaryCacheStore::init();

View file

@ -597,13 +597,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
} }
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) void LocalStore::registerDrvOutput(const Realisation & info)
{ {
auto state(_state.lock()); auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); // XXX: This ignores the references of the output because we can
// recompute them later from the drv and the references of the associated
// store path, but doing so is both inefficient and fragile.
return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath);
} }
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
{ {
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
state.stmts->AddDerivationOutput.use() state.stmts->AddDerivationOutput.use()
@ -653,7 +656,7 @@ uint64_t LocalStore::addValidPath(State & state,
/* Floating CA derivations have indeterminate output paths until /* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */ they are built, so don't register anything in that case */
if (i.second.second) if (i.second.second)
linkDeriverToPath(state, id, i.first, *i.second.second); registerDrvOutput_(state, id, i.first, *i.second.second);
} }
} }
@ -1612,5 +1615,13 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
} }
} }
std::optional<const DrvOutputInfo> LocalStore::queryDrvOutputInfo(const DrvOutputId& id) {
auto outputPath = queryOutputPathOf(id.drvPath, id.outputName);
if (!(outputPath && isValidPath(*outputPath)))
return std::nullopt;
else
return {DrvOutputInfo{
.outPath = *outputPath,
}};
} }
} // namespace nix

View file

@ -208,6 +208,13 @@ public:
garbage until it exceeds maxFree. */ garbage until it exceeds maxFree. */
void autoGC(bool sync = true); void autoGC(bool sync = true);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override;
void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
private: private:
int getSchema(); int getSchema();
@ -276,11 +283,6 @@ private:
specified by the secret-key-files option. */ specified by the secret-key-files option. */
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; } Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override; void createUser(const std::string & userName, uid_t userId) override;

View file

@ -0,0 +1,72 @@
#include "realisation.hh"
#include "store-api.hh"
namespace nix {
MakeError(InvalidDerivationOutputId, Error);
DrvOutput DrvOutput::parse(const std::string &strRep) {
const auto &[rawPath, outputs] = parsePathWithOutputs(strRep);
if (outputs.size() != 1)
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
return DrvOutput{
.drvPath = StorePath(rawPath),
.outputName = *outputs.begin(),
};
}
std::string DrvOutput::to_string() const {
return std::string(drvPath.to_string()) + "!" + outputName;
}
std::string Realisation::to_string() const {
std::string res;
res += "Id: " + id.to_string() + '\n';
res += "OutPath: " + std::string(outPath.to_string()) + '\n';
return res;
}
Realisation Realisation::parse(const std::string & s, const std::string & whence)
{
// XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out
auto corrupt = [&]() {
return Error("Drv output info file '%1%' is corrupt", whence);
};
std::optional<DrvOutput> id;
std::optional<StorePath> outPath;
size_t pos = 0;
while (pos < s.size()) {
size_t colon = s.find(':', pos);
if (colon == std::string::npos) throw corrupt();
std::string name(s, pos, colon - pos);
size_t eol = s.find('\n', colon + 2);
if (eol == std::string::npos) throw corrupt();
std::string value(s, colon + 2, eol - colon - 2);
if (name == "Id")
id = DrvOutput::parse(value);
if (name == "OutPath")
outPath = StorePath(value);
pos = eol + 1;
}
if (!outPath) corrupt();
if (!id) corrupt();
return Realisation {
.id = *id,
.outPath = *outPath,
};
}
} // namespace nix

View file

@ -0,0 +1,34 @@
#pragma once
#include "path.hh"
namespace nix {
struct DrvOutput {
StorePath drvPath;
std::string outputName;
std::string to_string() const;
static DrvOutput parse(const std::string &);
bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
private:
// Just to make comparison operators easier to write
std::pair<StorePath, std::string> to_pair() const
{ return std::make_pair(drvPath, outputName); }
};
struct Realisation {
DrvOutput id;
StorePath outPath;
std::string to_string() const;
static Realisation parse(const std::string & s, const std::string & whence);
};
typedef std::map<DrvOutput, Realisation> DrvOutputs;
}

View file

@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
} }
void RemoteStore::registerDrvOutput(const Realisation & info)
{
auto conn(getConnection());
conn->to << wopRegisterDrvOutput;
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
conn.processStderr();
}
std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
{
auto conn(getConnection());
conn->to << wopQueryRealisation;
conn->to << id.to_string();
conn.processStderr();
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
if (outPaths.empty())
return std::nullopt;
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
}
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
{ {

View file

@ -81,6 +81,10 @@ public:
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override; void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include "realisation.hh"
#include "path.hh" #include "path.hh"
#include "hash.hh" #include "hash.hh"
#include "content-address.hh" #include "content-address.hh"
@ -396,6 +397,8 @@ protected:
public: public:
virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
/* Queries the set of incoming FS references for a store path. /* Queries the set of incoming FS references for a store path.
The result is not cleared. */ The result is not cleared. */
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@ -468,6 +471,18 @@ public:
virtual StorePath addTextToStore(const string & name, const string & s, virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0; const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
/**
* Add a mapping indicating that `deriver!outputName` maps to the output path
* `output`.
*
* This is redundant for known-input-addressed and fixed-output derivations
* as this information is already present in the drv file, but necessary for
* floating-ca derivations and their dependencies as there's no way to
* retrieve this information otherwise.
*/
virtual void registerDrvOutput(const Realisation & output)
{ unsupported("registerDrvOutput"); }
/* Write a NAR dump of a store path. */ /* Write a NAR dump of a store path. */
virtual void narFromPath(const StorePath & path, Sink & sink) = 0; virtual void narFromPath(const StorePath & path, Sink & sink) = 0;

View file

@ -1,5 +1,8 @@
#pragma once #pragma once
#include "store-api.hh"
#include "serialise.hh"
namespace nix { namespace nix {
@ -50,6 +53,8 @@ typedef enum {
wopAddToStoreNar = 39, wopAddToStoreNar = 39,
wopQueryMissing = 40, wopQueryMissing = 40,
wopQueryDerivationOutputMap = 41, wopQueryDerivationOutputMap = 41,
wopRegisterDrvOutput = 42,
wopQueryRealisation = 43,
} WorkerOp; } WorkerOp;

View file

@ -55,7 +55,8 @@ testNixCommand () {
nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
} }
testRemoteCache # Disabled until we have it properly working
# testRemoteCache
testDeterministicCA testDeterministicCA
testCutoff testCutoff
testGC testGC