Merge remote-tracking branch 'origin/master' into cli-guideline
This commit is contained in:
Rok Garbas
commit
bdd05a1730
|
|
@ -1107,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
|||
// Shouldn't happen as the toplevel derivation is not CA.
|
||||
assert(false);
|
||||
},
|
||||
[&](UnknownHashes) {
|
||||
[&](DeferredHash _) {
|
||||
for (auto & i : outputs) {
|
||||
drv.outputs.insert_or_assign(i,
|
||||
DerivationOutput {
|
||||
|
|
@ -1621,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({
|
|||
static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
string s = state.forceStringNoCtx(*args[0], pos);
|
||||
parseJSON(state, s, v);
|
||||
try {
|
||||
parseJSON(state, s, v);
|
||||
} catch (JSONParseError &e) {
|
||||
e.addTrace(pos, "while decoding a JSON string");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
static RegisterPrimOp primop_fromJSON({
|
||||
|
|
|
|||
|
|
@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
|
|||
if (!repair && isValidPath(path))
|
||||
return path;
|
||||
|
||||
auto source = StringSource { s };
|
||||
StringSink sink;
|
||||
dumpString(s, sink);
|
||||
auto source = StringSource { *sink.s };
|
||||
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
|
||||
ValidPathInfo info { path, nar.first };
|
||||
info.narSize = nar.second;
|
||||
|
|
@ -443,6 +445,24 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
|
|||
})->path;
|
||||
}
|
||||
|
||||
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
|
||||
{
|
||||
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||
auto rawOutputInfo = getFile(outputInfoFilePath);
|
||||
|
||||
if (rawOutputInfo) {
|
||||
return {Realisation::fromJSON(
|
||||
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
|
||||
} else {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
|
||||
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
|
||||
upsertFile(filePath, info.toJSON().dump(), "application/json");
|
||||
}
|
||||
|
||||
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ private:
|
|||
|
||||
protected:
|
||||
|
||||
// The prefix under which realisation infos will be stored
|
||||
const std::string realisationsPrefix = "/realisations";
|
||||
|
||||
BinaryCacheStore(const Params & params);
|
||||
|
||||
public:
|
||||
|
|
@ -99,6 +102,10 @@ public:
|
|||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
||||
void registerDrvOutput(const Realisation & info) override;
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
|
||||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
|
|
|
|||
|
|
@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised()
|
|||
if (useDerivation) {
|
||||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
||||
|
||||
if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|
||||
|| fullDrv.type() == DerivationType::DeferredInputAddressed) {
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
|
||||
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|
||||
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
|
||||
/* We are be able to resolve this derivation based on the
|
||||
now-known results of dependencies. If so, we become a stub goal
|
||||
aliasing that resolved derivation goal */
|
||||
|
|
@ -503,9 +504,6 @@ void DerivationGoal::inputsRealised()
|
|||
Derivation drvResolved { *std::move(attempt) };
|
||||
|
||||
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
||||
/* Add to memotable to speed up downstream goal's queries with the
|
||||
original derivation. */
|
||||
drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
|
||||
|
||||
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
|
|
@ -2094,6 +2092,16 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
|
|||
/* Nothing to be done; 'path' must already be valid. */
|
||||
}
|
||||
|
||||
void registerDrvOutput(const Realisation & info) override
|
||||
// XXX: This should probably be allowed as a no-op if the realisation
|
||||
// corresponds to an allowed derivation
|
||||
{ throw Error("registerDrvOutput"); }
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
|
||||
// XXX: This should probably be allowed if the realisation corresponds to
|
||||
// an allowed derivation
|
||||
{ throw Error("queryRealisation"); }
|
||||
|
||||
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
|
||||
{
|
||||
if (buildMode != bmNormal) throw Error("unsupported build mode");
|
||||
|
|
@ -2872,6 +2880,8 @@ void DerivationGoal::registerOutputs()
|
|||
for (auto & i : drv->outputsAndOptPaths(worker.store)) {
|
||||
if (!i.second.second || !worker.store.isValidPath(*i.second.second))
|
||||
allValid = false;
|
||||
else
|
||||
finalOutputs.insert_or_assign(i.first, *i.second.second);
|
||||
}
|
||||
if (allValid) return;
|
||||
}
|
||||
|
|
@ -3377,21 +3387,14 @@ void DerivationGoal::registerOutputs()
|
|||
means it's safe to link the derivation to the output hash. We must do
|
||||
that for floating CA derivations, which otherwise couldn't be cached,
|
||||
but it's fine to do in all cases. */
|
||||
bool isCaFloating = drv->type() == DerivationType::CAFloating;
|
||||
|
||||
auto drvPathResolved = drvPath;
|
||||
if (!useDerivation && isCaFloating) {
|
||||
/* Once a floating CA derivations reaches this point, it
|
||||
must already be resolved, so we don't bother trying to
|
||||
downcast drv to get would would just be an empty
|
||||
inputDrvs field. */
|
||||
Derivation drv2 { *drv };
|
||||
drvPathResolved = writeDerivation(worker.store, drv2);
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
auto outputHashes = staticOutputHashes(worker.store, *drv);
|
||||
for (auto& [outputName, newInfo] : infos)
|
||||
worker.store.registerDrvOutput(Realisation{
|
||||
.id = DrvOutput{outputHashes.at(outputName), outputName},
|
||||
.outPath = newInfo.path});
|
||||
}
|
||||
|
||||
if (useDerivation || isCaFloating)
|
||||
for (auto & [outputName, newInfo] : infos)
|
||||
worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
11
src/libstore/ca-specific-schema.sql
Normal file
11
src/libstore/ca-specific-schema.sql
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
-- Extension of the sql schema for content-addressed derivations.
|
||||
-- Won't be loaded unless the experimental feature `ca-derivations`
|
||||
-- is enabled
|
||||
|
||||
create table if not exists Realisations (
|
||||
drvPath text not null,
|
||||
outputName text not null, -- symbolic output id, usually "out"
|
||||
outputPath integer not null,
|
||||
primary key (drvPath, outputName),
|
||||
foreign key (outputPath) references ValidPaths(id) on delete cascade
|
||||
);
|
||||
|
|
@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
break;
|
||||
}
|
||||
|
||||
case wopRegisterDrvOutput: {
|
||||
logger->startWork();
|
||||
auto outputId = DrvOutput::parse(readString(from));
|
||||
auto outputPath = StorePath(readString(from));
|
||||
auto resolvedDrv = StorePath(readString(from));
|
||||
store->registerDrvOutput(Realisation{
|
||||
.id = outputId, .outPath = outputPath});
|
||||
logger->stopWork();
|
||||
break;
|
||||
}
|
||||
|
||||
case wopQueryRealisation: {
|
||||
logger->startWork();
|
||||
auto outputId = DrvOutput::parse(readString(from));
|
||||
auto info = store->queryRealisation(outputId);
|
||||
logger->stopWork();
|
||||
std::set<StorePath> outPaths;
|
||||
if (info) outPaths.insert(info->outPath);
|
||||
worker_proto::write(*store, to, outPaths);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw Error("invalid operation %1%", op);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
|
|||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
{
|
||||
bool isDeferred = false;
|
||||
/* Return a fixed hash for fixed-output derivations. */
|
||||
switch (drv.type()) {
|
||||
case DerivationType::CAFloating:
|
||||
return UnknownHashes {};
|
||||
case DerivationType::CAFixed: {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & i : drv.outputs) {
|
||||
|
|
@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
}
|
||||
return outputHashes;
|
||||
}
|
||||
case DerivationType::CAFloating:
|
||||
isDeferred = true;
|
||||
break;
|
||||
case DerivationType::InputAddressed:
|
||||
break;
|
||||
case DerivationType::DeferredInputAddressed:
|
||||
|
|
@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
calls to this function. */
|
||||
std::map<std::string, StringSet> inputs2;
|
||||
for (auto & i : drv.inputDrvs) {
|
||||
bool hasUnknownHash = false;
|
||||
const auto & res = pathDerivationModulo(store, i.first);
|
||||
std::visit(overloaded {
|
||||
// Regular non-CA derivation, replace derivation
|
||||
[&](Hash drvHash) {
|
||||
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
|
||||
},
|
||||
[&](DeferredHash deferredHash) {
|
||||
isDeferred = true;
|
||||
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
|
||||
},
|
||||
// CA derivation's output hashes
|
||||
[&](CaOutputHashes outputHashes) {
|
||||
std::set<std::string> justOut = { "out" };
|
||||
|
|
@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
justOut);
|
||||
}
|
||||
},
|
||||
[&](UnknownHashes) {
|
||||
hasUnknownHash = true;
|
||||
},
|
||||
}, res);
|
||||
if (hasUnknownHash) {
|
||||
return UnknownHashes {};
|
||||
}
|
||||
}
|
||||
|
||||
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
|
||||
if (isDeferred)
|
||||
return DeferredHash { hash };
|
||||
else
|
||||
return hash;
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
|
||||
{
|
||||
std::map<std::string, Hash> res;
|
||||
std::visit(overloaded {
|
||||
[&](Hash drvHash) {
|
||||
for (auto & outputName : drv.outputNames()) {
|
||||
res.insert({outputName, drvHash});
|
||||
}
|
||||
},
|
||||
[&](DeferredHash deferredHash) {
|
||||
for (auto & outputName : drv.outputNames()) {
|
||||
res.insert({outputName, deferredHash.hash});
|
||||
}
|
||||
},
|
||||
[&](CaOutputHashes outputHashes) {
|
||||
res = outputHashes;
|
||||
},
|
||||
}, hashDerivationModulo(store, drv, true));
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -719,10 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
|
|||
|
||||
}
|
||||
|
||||
|
||||
Sync<DrvPathResolutions> drvPathResolutions;
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
|
||||
std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
|
||||
BasicDerivation resolved { *this };
|
||||
|
||||
// Input paths that we'll want to rewrite in the derivation
|
||||
|
|
@ -748,4 +771,34 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
|
|||
return resolved;
|
||||
}
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
|
||||
{
|
||||
auto drvPath = writeDerivation(store, *this, NoRepair, false);
|
||||
return Derivation::tryResolve(store, drvPath);
|
||||
}
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
|
||||
{
|
||||
// This is quite dirty and leaky, but will disappear once #4340 is merged
|
||||
static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
|
||||
|
||||
{
|
||||
auto resolutions = resolutionsCache.lock();
|
||||
auto resolvedDrvIter = resolutions->find(drvPath);
|
||||
if (resolvedDrvIter != resolutions->end()) {
|
||||
auto & [_, resolvedDrv] = *resolvedDrvIter;
|
||||
return *resolvedDrv;
|
||||
}
|
||||
}
|
||||
|
||||
/* Try resolve drv and use that path instead. */
|
||||
auto drv = store.readDerivation(drvPath);
|
||||
auto attempt = drv.tryResolveUncached(store);
|
||||
if (!attempt)
|
||||
return std::nullopt;
|
||||
/* Store in memo table. */
|
||||
resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
|
||||
return *attempt;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ namespace nix {
|
|||
/* The traditional non-fixed-output derivation type. */
|
||||
struct DerivationOutputInputAddressed
|
||||
{
|
||||
/* Will need to become `std::optional<StorePath>` once input-addressed
|
||||
derivations are allowed to depend on cont-addressed derivations */
|
||||
StorePath path;
|
||||
};
|
||||
|
||||
|
|
@ -140,10 +138,14 @@ struct Derivation : BasicDerivation
|
|||
|
||||
2. Input placeholders are replaced with realized input store paths. */
|
||||
std::optional<BasicDerivation> tryResolve(Store & store);
|
||||
static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
|
||||
|
||||
Derivation() = default;
|
||||
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
|
||||
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
|
||||
|
||||
private:
|
||||
std::optional<BasicDerivation> tryResolveUncached(Store & store);
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -174,12 +176,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
|
|||
// whose output hashes are always known since they are fixed up-front.
|
||||
typedef std::map<std::string, Hash> CaOutputHashes;
|
||||
|
||||
struct UnknownHashes {};
|
||||
struct DeferredHash { Hash hash; };
|
||||
|
||||
typedef std::variant<
|
||||
Hash, // regular DRV normalized hash
|
||||
CaOutputHashes, // Fixed-output derivation hashes
|
||||
UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies
|
||||
DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
|
||||
> DrvHashModulo;
|
||||
|
||||
/* Returns hashes with the details of fixed-output subderivations
|
||||
|
|
@ -207,22 +209,18 @@ typedef std::variant<
|
|||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
|
||||
/*
|
||||
Return a map associating each output to a hash that uniquely identifies its
|
||||
derivation (modulo the self-references).
|
||||
*/
|
||||
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
|
||||
|
||||
/* Memoisation of hashDerivationModulo(). */
|
||||
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
extern Sync<DrvHashes> drvHashes;
|
||||
|
||||
/* Memoisation of `readDerivation(..).resove()`. */
|
||||
typedef std::map<
|
||||
StorePath,
|
||||
std::optional<StorePath>
|
||||
> DrvPathResolutions;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
// FIXME: arguably overlaps with hashDerivationModulo memo table.
|
||||
extern Sync<DrvPathResolutions> drvPathResolutions;
|
||||
|
||||
bool wantOutput(const string & output, const std::set<string> & wanted);
|
||||
|
||||
struct Source;
|
||||
|
|
|
|||
|
|
@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
|
|||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode) override
|
||||
{ unsupported("buildDerivation"); }
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
|
||||
{ unsupported("queryRealisation"); }
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
|
||||
|
|
|
|||
|
|
@ -333,6 +333,10 @@ public:
|
|||
auto conn(connections->get());
|
||||
return conn->remoteVersion;
|
||||
}
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
|
||||
// TODO: Implement
|
||||
{ unsupported("queryRealisation"); }
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ protected:
|
|||
void LocalBinaryCacheStore::init()
|
||||
{
|
||||
createDirs(binaryCacheDir + "/nar");
|
||||
createDirs(binaryCacheDir + realisationsPrefix);
|
||||
if (writeDebugInfo)
|
||||
createDirs(binaryCacheDir + "/debuginfo");
|
||||
BinaryCacheStore::init();
|
||||
|
|
|
|||
|
|
@ -42,6 +42,61 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
struct LocalStore::State::Stmts {
|
||||
/* Some precompiled SQLite statements. */
|
||||
SQLiteStmt RegisterValidPath;
|
||||
SQLiteStmt UpdatePathInfo;
|
||||
SQLiteStmt AddReference;
|
||||
SQLiteStmt QueryPathInfo;
|
||||
SQLiteStmt QueryReferences;
|
||||
SQLiteStmt QueryReferrers;
|
||||
SQLiteStmt InvalidatePath;
|
||||
SQLiteStmt AddDerivationOutput;
|
||||
SQLiteStmt RegisterRealisedOutput;
|
||||
SQLiteStmt QueryValidDerivers;
|
||||
SQLiteStmt QueryDerivationOutputs;
|
||||
SQLiteStmt QueryRealisedOutput;
|
||||
SQLiteStmt QueryAllRealisedOutputs;
|
||||
SQLiteStmt QueryPathFromHashPart;
|
||||
SQLiteStmt QueryValidPaths;
|
||||
};
|
||||
|
||||
int getSchema(Path schemaPath)
|
||||
{
|
||||
int curSchema = 0;
|
||||
if (pathExists(schemaPath)) {
|
||||
string s = readFile(schemaPath);
|
||||
if (!string2Int(s, curSchema))
|
||||
throw Error("'%1%' is corrupt", schemaPath);
|
||||
}
|
||||
return curSchema;
|
||||
}
|
||||
|
||||
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
|
||||
{
|
||||
const int nixCASchemaVersion = 1;
|
||||
int curCASchema = getSchema(schemaPath);
|
||||
if (curCASchema != nixCASchemaVersion) {
|
||||
if (curCASchema > nixCASchemaVersion) {
|
||||
throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
|
||||
curCASchema, nixCASchemaVersion);
|
||||
}
|
||||
|
||||
if (!lockFile(lockFd.get(), ltWrite, false)) {
|
||||
printInfo("waiting for exclusive access to the Nix store for ca drvs...");
|
||||
lockFile(lockFd.get(), ltWrite, true);
|
||||
}
|
||||
|
||||
if (curCASchema == 0) {
|
||||
static const char schema[] =
|
||||
#include "ca-specific-schema.sql.gen.hh"
|
||||
;
|
||||
db.exec(schema);
|
||||
}
|
||||
writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
|
||||
lockFile(lockFd.get(), ltRead, true);
|
||||
}
|
||||
}
|
||||
|
||||
LocalStore::LocalStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
|
|
@ -60,6 +115,7 @@ LocalStore::LocalStore(const Params & params)
|
|||
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
|
||||
{
|
||||
auto state(_state.lock());
|
||||
state->stmts = std::make_unique<State::Stmts>();
|
||||
|
||||
/* Create missing state directories if they don't already exist. */
|
||||
createDirs(realStoreDir);
|
||||
|
|
@ -222,32 +278,58 @@ LocalStore::LocalStore(const Params & params)
|
|||
|
||||
else openDB(*state, false);
|
||||
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
|
||||
}
|
||||
|
||||
/* Prepare SQL statements. */
|
||||
state->stmtRegisterValidPath.create(state->db,
|
||||
state->stmts->RegisterValidPath.create(state->db,
|
||||
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
|
||||
state->stmtUpdatePathInfo.create(state->db,
|
||||
state->stmts->UpdatePathInfo.create(state->db,
|
||||
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
|
||||
state->stmtAddReference.create(state->db,
|
||||
state->stmts->AddReference.create(state->db,
|
||||
"insert or replace into Refs (referrer, reference) values (?, ?);");
|
||||
state->stmtQueryPathInfo.create(state->db,
|
||||
state->stmts->QueryPathInfo.create(state->db,
|
||||
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
|
||||
state->stmtQueryReferences.create(state->db,
|
||||
state->stmts->QueryReferences.create(state->db,
|
||||
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
|
||||
state->stmtQueryReferrers.create(state->db,
|
||||
state->stmts->QueryReferrers.create(state->db,
|
||||
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
|
||||
state->stmtInvalidatePath.create(state->db,
|
||||
state->stmts->InvalidatePath.create(state->db,
|
||||
"delete from ValidPaths where path = ?;");
|
||||
state->stmtAddDerivationOutput.create(state->db,
|
||||
state->stmts->AddDerivationOutput.create(state->db,
|
||||
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
|
||||
state->stmtQueryValidDerivers.create(state->db,
|
||||
state->stmts->QueryValidDerivers.create(state->db,
|
||||
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
|
||||
state->stmtQueryDerivationOutputs.create(state->db,
|
||||
state->stmts->QueryDerivationOutputs.create(state->db,
|
||||
"select id, path from DerivationOutputs where drv = ?;");
|
||||
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
|
||||
// ensure efficient lookup.
|
||||
state->stmtQueryPathFromHashPart.create(state->db,
|
||||
state->stmts->QueryPathFromHashPart.create(state->db,
|
||||
"select path from ValidPaths where path >= ? limit 1;");
|
||||
state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths");
|
||||
state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
state->stmts->RegisterRealisedOutput.create(state->db,
|
||||
R"(
|
||||
insert or replace into Realisations (drvPath, outputName, outputPath)
|
||||
values (?, ?, (select id from ValidPaths where path = ?))
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryRealisedOutput.create(state->db,
|
||||
R"(
|
||||
select Output.path from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
where drvPath = ? and outputName = ?
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryAllRealisedOutputs.create(state->db,
|
||||
R"(
|
||||
select outputName, Output.path from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
where drvPath = ?
|
||||
;
|
||||
)");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -285,16 +367,7 @@ std::string LocalStore::getUri()
|
|||
|
||||
|
||||
int LocalStore::getSchema()
|
||||
{
|
||||
int curSchema = 0;
|
||||
if (pathExists(schemaPath)) {
|
||||
string s = readFile(schemaPath);
|
||||
if (!string2Int(s, curSchema))
|
||||
throw Error("'%1%' is corrupt", schemaPath);
|
||||
}
|
||||
return curSchema;
|
||||
}
|
||||
|
||||
{ return nix::getSchema(schemaPath); }
|
||||
|
||||
void LocalStore::openDB(State & state, bool create)
|
||||
{
|
||||
|
|
@ -581,16 +654,22 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
|
||||
void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
auto state(_state.lock());
|
||||
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
|
||||
retrySQLite<void>([&]() {
|
||||
state->stmts->RegisterRealisedOutput.use()
|
||||
(info.id.strHash())
|
||||
(info.id.outputName)
|
||||
(printStorePath(info.outPath))
|
||||
.exec();
|
||||
});
|
||||
}
|
||||
|
||||
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
|
||||
void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
state.stmtAddDerivationOutput.use()
|
||||
state.stmts->AddDerivationOutput.use()
|
||||
(deriver)
|
||||
(outputName)
|
||||
(printStorePath(output))
|
||||
|
|
@ -607,7 +686,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
|
||||
printStorePath(info.path));
|
||||
|
||||
state.stmtRegisterValidPath.use()
|
||||
state.stmts->RegisterValidPath.use()
|
||||
(printStorePath(info.path))
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.registrationTime == 0 ? time(0) : info.registrationTime)
|
||||
|
|
@ -637,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
/* Floating CA derivations have indeterminate output paths until
|
||||
they are built, so don't register anything in that case */
|
||||
if (i.second.second)
|
||||
linkDeriverToPath(state, id, i.first, *i.second.second);
|
||||
cacheDrvOutputMapping(state, id, i.first, *i.second.second);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -659,7 +738,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
|
|||
auto state(_state.lock());
|
||||
|
||||
/* Get the path info. */
|
||||
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
|
||||
auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
|
||||
|
||||
if (!useQueryPathInfo.next())
|
||||
return std::shared_ptr<ValidPathInfo>();
|
||||
|
|
@ -679,7 +758,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
|
|||
|
||||
info->registrationTime = useQueryPathInfo.getInt(2);
|
||||
|
||||
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
|
||||
auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
|
||||
if (s) info->deriver = parseStorePath(s);
|
||||
|
||||
/* Note that narSize = NULL yields 0. */
|
||||
|
|
@ -687,14 +766,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
|
|||
|
||||
info->ultimate = useQueryPathInfo.getInt(5) == 1;
|
||||
|
||||
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
|
||||
s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
|
||||
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
|
||||
|
||||
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
|
||||
s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
|
||||
if (s) info->ca = parseContentAddressOpt(s);
|
||||
|
||||
/* Get the references. */
|
||||
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
|
||||
auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
|
||||
|
||||
while (useQueryReferences.next())
|
||||
info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
|
||||
|
|
@ -709,7 +788,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
|
|||
/* Update path info in the database. */
|
||||
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
|
||||
{
|
||||
state.stmtUpdatePathInfo.use()
|
||||
state.stmts->UpdatePathInfo.use()
|
||||
(info.narSize, info.narSize != 0)
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.ultimate ? 1 : 0, info.ultimate)
|
||||
|
|
@ -722,7 +801,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
|
|||
|
||||
uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
|
||||
{
|
||||
auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
|
||||
auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
|
||||
if (!use.next())
|
||||
throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
||||
return use.getInt(0);
|
||||
|
|
@ -731,7 +810,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
|
|||
|
||||
bool LocalStore::isValidPath_(State & state, const StorePath & path)
|
||||
{
|
||||
return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
|
||||
return state.stmts->QueryPathInfo.use()(printStorePath(path)).next();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -757,7 +836,7 @@ StorePathSet LocalStore::queryAllValidPaths()
|
|||
{
|
||||
return retrySQLite<StorePathSet>([&]() {
|
||||
auto state(_state.lock());
|
||||
auto use(state->stmtQueryValidPaths.use());
|
||||
auto use(state->stmts->QueryValidPaths.use());
|
||||
StorePathSet res;
|
||||
while (use.next()) res.insert(parseStorePath(use.getStr(0)));
|
||||
return res;
|
||||
|
|
@ -767,7 +846,7 @@ StorePathSet LocalStore::queryAllValidPaths()
|
|||
|
||||
void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
|
||||
{
|
||||
auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
|
||||
auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path)));
|
||||
|
||||
while (useQueryReferrers.next())
|
||||
referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
|
||||
|
|
@ -788,7 +867,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
|
|||
return retrySQLite<StorePathSet>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
|
||||
auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
|
||||
|
||||
StorePathSet derivers;
|
||||
while (useQueryValidDerivers.next())
|
||||
|
|
@ -799,69 +878,38 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
|
|||
}
|
||||
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
|
||||
std::map<std::string, std::optional<StorePath>>
|
||||
LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
|
||||
{
|
||||
auto path = path_;
|
||||
std::map<std::string, std::optional<StorePath>> outputs;
|
||||
Derivation drv = readDerivation(path);
|
||||
for (auto & [outName, _] : drv.outputs) {
|
||||
outputs.insert_or_assign(outName, std::nullopt);
|
||||
}
|
||||
bool haveCached = false;
|
||||
{
|
||||
auto resolutions = drvPathResolutions.lock();
|
||||
auto resolvedPathOptIter = resolutions->find(path);
|
||||
if (resolvedPathOptIter != resolutions->end()) {
|
||||
auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
|
||||
if (resolvedPathOpt)
|
||||
path = *resolvedPathOpt;
|
||||
haveCached = true;
|
||||
}
|
||||
}
|
||||
/* can't just use else-if instead of `!haveCached` because we need to unlock
|
||||
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
|
||||
if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) {
|
||||
/* Try resolve drv and use that path instead. */
|
||||
auto attempt = drv.tryResolve(*this);
|
||||
if (!attempt)
|
||||
/* If we cannot resolve the derivation, we cannot have any path
|
||||
assigned so we return the map of all std::nullopts. */
|
||||
return outputs;
|
||||
/* Just compute store path */
|
||||
auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
|
||||
/* Store in memo table. */
|
||||
/* FIXME: memo logic should not be local-store specific, should have
|
||||
wrapper-method instead. */
|
||||
drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
|
||||
path = std::move(pathResolved);
|
||||
}
|
||||
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
|
||||
auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> outputs;
|
||||
uint64_t drvId;
|
||||
try {
|
||||
drvId = queryValidPathId(*state, path);
|
||||
} catch (InvalidPath &) {
|
||||
/* FIXME? if the derivation doesn't exist, we cannot have a mapping
|
||||
for it. */
|
||||
return outputs;
|
||||
}
|
||||
|
||||
auto useQueryDerivationOutputs {
|
||||
state->stmtQueryDerivationOutputs.use()
|
||||
(drvId)
|
||||
};
|
||||
|
||||
while (useQueryDerivationOutputs.next())
|
||||
drvId = queryValidPathId(*state, path);
|
||||
auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
|
||||
while (use.next())
|
||||
outputs.insert_or_assign(
|
||||
useQueryDerivationOutputs.getStr(0),
|
||||
parseStorePath(useQueryDerivationOutputs.getStr(1))
|
||||
);
|
||||
use.getStr(0), parseStorePath(use.getStr(1)));
|
||||
|
||||
return outputs;
|
||||
});
|
||||
}
|
||||
|
||||
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
|
||||
return outputs;
|
||||
|
||||
auto drv = readInvalidDerivation(path);
|
||||
auto drvHashes = staticOutputHashes(*this, drv);
|
||||
for (auto& [outputName, hash] : drvHashes) {
|
||||
auto realisation = queryRealisation(DrvOutput{hash, outputName});
|
||||
if (realisation)
|
||||
outputs.insert_or_assign(outputName, realisation->outPath);
|
||||
else
|
||||
outputs.insert_or_assign(outputName, std::nullopt);
|
||||
}
|
||||
|
||||
return outputs;
|
||||
}
|
||||
|
||||
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
|
||||
{
|
||||
|
|
@ -872,11 +920,11 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
|
|||
return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
|
||||
auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
|
||||
|
||||
if (!useQueryPathFromHashPart.next()) return {};
|
||||
|
||||
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
|
||||
const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0);
|
||||
if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
|
||||
return parseStorePath(s);
|
||||
return {};
|
||||
|
|
@ -990,7 +1038,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||
for (auto & [_, i] : infos) {
|
||||
auto referrer = queryValidPathId(*state, i.path);
|
||||
for (auto & j : i.references)
|
||||
state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||
}
|
||||
|
||||
/* Check that the derivation outputs are correct. We can't do
|
||||
|
|
@ -1030,7 +1078,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
|
|||
{
|
||||
debug("invalidating path '%s'", printStorePath(path));
|
||||
|
||||
state.stmtInvalidatePath.use()(printStorePath(path)).exec();
|
||||
state.stmts->InvalidatePath.use()(printStorePath(path)).exec();
|
||||
|
||||
/* Note that the foreign key constraints on the Refs table take
|
||||
care of deleting the references entries for `path'. */
|
||||
|
|
@ -1596,5 +1644,18 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
std::optional<const Realisation> LocalStore::queryRealisation(
|
||||
const DrvOutput& id) {
|
||||
typedef std::optional<const Realisation> Ret;
|
||||
return retrySQLite<Ret>([&]() -> Ret {
|
||||
auto state(_state.lock());
|
||||
auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
|
||||
id.outputName));
|
||||
if (!use.next())
|
||||
return std::nullopt;
|
||||
auto outputPath = parseStorePath(use.getStr(0));
|
||||
return Ret{
|
||||
Realisation{.id = id, .outPath = outputPath}};
|
||||
});
|
||||
}
|
||||
} // namespace nix
|
||||
|
|
|
|||
|
|
@ -55,19 +55,8 @@ private:
|
|||
/* The SQLite database object. */
|
||||
SQLite db;
|
||||
|
||||
/* Some precompiled SQLite statements. */
|
||||
SQLiteStmt stmtRegisterValidPath;
|
||||
SQLiteStmt stmtUpdatePathInfo;
|
||||
SQLiteStmt stmtAddReference;
|
||||
SQLiteStmt stmtQueryPathInfo;
|
||||
SQLiteStmt stmtQueryReferences;
|
||||
SQLiteStmt stmtQueryReferrers;
|
||||
SQLiteStmt stmtInvalidatePath;
|
||||
SQLiteStmt stmtAddDerivationOutput;
|
||||
SQLiteStmt stmtQueryValidDerivers;
|
||||
SQLiteStmt stmtQueryDerivationOutputs;
|
||||
SQLiteStmt stmtQueryPathFromHashPart;
|
||||
SQLiteStmt stmtQueryValidPaths;
|
||||
struct Stmts;
|
||||
std::unique_ptr<Stmts> stmts;
|
||||
|
||||
/* The file to which we write our temporary roots. */
|
||||
AutoCloseFD fdTempRoots;
|
||||
|
|
@ -138,7 +127,7 @@ public:
|
|||
|
||||
StorePathSet queryValidDerivers(const StorePath & path) override;
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
|
||||
std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
|
||||
|
||||
|
|
@ -219,6 +208,13 @@ public:
|
|||
garbage until it exceeds maxFree. */
|
||||
void autoGC(bool sync = true);
|
||||
|
||||
/* Register the store path 'output' as the output named 'outputName' of
|
||||
derivation 'deriver'. */
|
||||
void registerDrvOutput(const Realisation & info) override;
|
||||
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
|
||||
|
||||
private:
|
||||
|
||||
int getSchema();
|
||||
|
|
@ -287,11 +283,6 @@ private:
|
|||
specified by the ‘secret-key-files’ option. */
|
||||
void signPathInfo(ValidPathInfo & info);
|
||||
|
||||
/* Register the store path 'output' as the output named 'outputName' of
|
||||
derivation 'deriver'. */
|
||||
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
|
||||
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
|
||||
|
||||
Path getRealStoreDir() override { return realStoreDir; }
|
||||
|
||||
void createUser(const std::string & userName, uid_t userId) override;
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),)
|
|||
libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
|
||||
endif
|
||||
|
||||
$(d)/local-store.cc: $(d)/schema.sql.gen.hh
|
||||
$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
||||
|
||||
$(d)/build.cc:
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ $(d)/build.cc:
|
|||
@echo ')foo"' >> $@.tmp
|
||||
@mv $@.tmp $@
|
||||
|
||||
clean-files += $(d)/schema.sql.gen.hh
|
||||
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
||||
|
||||
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
|
||||
|
||||
|
|
|
|||
49
src/libstore/realisation.cc
Normal file
49
src/libstore/realisation.cc
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#include "realisation.hh"
|
||||
#include "store-api.hh"
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
MakeError(InvalidDerivationOutputId, Error);
|
||||
|
||||
DrvOutput DrvOutput::parse(const std::string &strRep) {
|
||||
size_t n = strRep.find("!");
|
||||
if (n == strRep.npos)
|
||||
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
|
||||
|
||||
return DrvOutput{
|
||||
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
|
||||
.outputName = strRep.substr(n+1),
|
||||
};
|
||||
}
|
||||
|
||||
std::string DrvOutput::to_string() const {
|
||||
return strHash() + "!" + outputName;
|
||||
}
|
||||
|
||||
nlohmann::json Realisation::toJSON() const {
|
||||
return nlohmann::json{
|
||||
{"id", id.to_string()},
|
||||
{"outPath", outPath.to_string()},
|
||||
};
|
||||
}
|
||||
|
||||
Realisation Realisation::fromJSON(
|
||||
const nlohmann::json& json,
|
||||
const std::string& whence) {
|
||||
auto getField = [&](std::string fieldName) -> std::string {
|
||||
auto fieldIterator = json.find(fieldName);
|
||||
if (fieldIterator == json.end())
|
||||
throw Error(
|
||||
"Drv output info file '%1%' is corrupt, missing field %2%",
|
||||
whence, fieldName);
|
||||
return *fieldIterator;
|
||||
};
|
||||
|
||||
return Realisation{
|
||||
.id = DrvOutput::parse(getField("id")),
|
||||
.outPath = StorePath(getField("outPath")),
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
39
src/libstore/realisation.hh
Normal file
39
src/libstore/realisation.hh
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#pragma once
|
||||
|
||||
#include "path.hh"
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct DrvOutput {
|
||||
// The hash modulo of the derivation
|
||||
Hash drvHash;
|
||||
std::string outputName;
|
||||
|
||||
std::string to_string() const;
|
||||
|
||||
std::string strHash() const
|
||||
{ return drvHash.to_string(Base16, true); }
|
||||
|
||||
static DrvOutput parse(const std::string &);
|
||||
|
||||
bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
|
||||
bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
|
||||
|
||||
private:
|
||||
// Just to make comparison operators easier to write
|
||||
std::pair<Hash, std::string> to_pair() const
|
||||
{ return std::make_pair(drvHash, outputName); }
|
||||
};
|
||||
|
||||
struct Realisation {
|
||||
DrvOutput id;
|
||||
StorePath outPath;
|
||||
|
||||
nlohmann::json toJSON() const;
|
||||
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
|
||||
};
|
||||
|
||||
typedef std::map<DrvOutput, Realisation> DrvOutputs;
|
||||
|
||||
}
|
||||
|
|
@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
|
|||
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
|
||||
}
|
||||
|
||||
void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopRegisterDrvOutput;
|
||||
conn->to << info.id.to_string();
|
||||
conn->to << std::string(info.outPath.to_string());
|
||||
conn.processStderr();
|
||||
}
|
||||
|
||||
std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopQueryRealisation;
|
||||
conn->to << id.to_string();
|
||||
conn.processStderr();
|
||||
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
|
||||
if (outPaths.empty())
|
||||
return std::nullopt;
|
||||
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -81,6 +81,10 @@ public:
|
|||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
||||
void registerDrvOutput(const Realisation & info) override;
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
|
||||
|
||||
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
|
||||
|
||||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
|
|
|
|||
|
|
@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
|
|||
dynamic_cast<std::stringstream &>(result.GetBody()).str());
|
||||
|
||||
} catch (S3Error & e) {
|
||||
if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
|
||||
if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
|
||||
(e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw;
|
||||
}
|
||||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@
|
|||
#include "archive.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
|
|
@ -364,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow()
|
|||
return std::chrono::steady_clock::now() < time_point + ttl;
|
||||
}
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
|
||||
{
|
||||
std::map<std::string, std::optional<StorePath>> outputs;
|
||||
auto drv = readInvalidDerivation(path);
|
||||
for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
|
||||
outputs.emplace(outputName, output.second);
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
|
||||
{
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
auto resolvedDrv = Derivation::tryResolve(*this, path);
|
||||
if (resolvedDrv) {
|
||||
auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
|
||||
if (isValidPath(resolvedDrvPath))
|
||||
return queryDerivationOutputMapNoResolve(resolvedDrvPath);
|
||||
}
|
||||
}
|
||||
return queryDerivationOutputMapNoResolve(path);
|
||||
}
|
||||
|
||||
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
|
||||
auto resp = queryPartialDerivationOutputMap(path);
|
||||
OutputPathMap result;
|
||||
|
|
@ -727,9 +752,17 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
|
|||
StorePathSet paths2;
|
||||
|
||||
for (auto & path : paths) {
|
||||
if (path.path.isDerivation())
|
||||
unsupported("buildPaths");
|
||||
paths2.insert(path.path);
|
||||
if (path.path.isDerivation()) {
|
||||
auto outPaths = queryPartialDerivationOutputMap(path.path);
|
||||
for (auto & outputName : path.outputs) {
|
||||
auto currentOutputPathIter = outPaths.find(outputName);
|
||||
if (currentOutputPathIter == outPaths.end() ||
|
||||
!currentOutputPathIter->second ||
|
||||
!isValidPath(*currentOutputPathIter->second))
|
||||
unsupported("buildPaths");
|
||||
}
|
||||
} else
|
||||
paths2.insert(path.path);
|
||||
}
|
||||
|
||||
if (queryValidPaths(paths2).size() != paths2.size())
|
||||
|
|
@ -1091,6 +1124,34 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
|
|||
}
|
||||
}
|
||||
|
||||
// The `parseURL` function supports both IPv6 URIs as defined in
|
||||
// RFC2732, but also pure addresses. The latter one is needed here to
|
||||
// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
|
||||
//
|
||||
// This function now ensures that a usable connection string is available:
|
||||
// * If the store to be opened is not an SSH store, nothing will be done.
|
||||
// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
|
||||
// needed to pass further flags), it
|
||||
// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
|
||||
// * If the URL looks like `root@::1` it will be left as-is.
|
||||
// * In any other case, the string will be left as-is.
|
||||
static std::string extractConnStr(const std::string &proto, const std::string &connStr)
|
||||
{
|
||||
if (proto.rfind("ssh") != std::string::npos) {
|
||||
std::smatch result;
|
||||
std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
|
||||
|
||||
if (std::regex_match(connStr, result, v6AddrRegex)) {
|
||||
if (result[1].matched) {
|
||||
return result.str(1) + result.str(3);
|
||||
}
|
||||
return result.str(3);
|
||||
}
|
||||
}
|
||||
|
||||
return connStr;
|
||||
}
|
||||
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
|
|
@ -1099,7 +1160,10 @@ ref<Store> openStore(const std::string & uri_,
|
|||
auto parsedUri = parseURL(uri_);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
|
||||
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
|
||||
auto baseURI = extractConnStr(
|
||||
parsedUri.scheme,
|
||||
parsedUri.authority.value_or("") + parsedUri.path
|
||||
);
|
||||
|
||||
for (auto implem : *Implementations::registered) {
|
||||
if (implem.uriSchemes.count(parsedUri.scheme)) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include "realisation.hh"
|
||||
#include "path.hh"
|
||||
#include "hash.hh"
|
||||
#include "content-address.hh"
|
||||
|
|
@ -396,6 +397,8 @@ protected:
|
|||
|
||||
public:
|
||||
|
||||
virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
|
||||
|
||||
/* Queries the set of incoming FS references for a store path.
|
||||
The result is not cleared. */
|
||||
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
|
||||
|
|
@ -413,8 +416,13 @@ public:
|
|||
/* Query the mapping outputName => outputPath for the given derivation. All
|
||||
outputs are mentioned so ones mising the mapping are mapped to
|
||||
`std::nullopt`. */
|
||||
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
|
||||
{ unsupported("queryPartialDerivationOutputMap"); }
|
||||
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
|
||||
|
||||
/*
|
||||
* Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
|
||||
* the derivation
|
||||
*/
|
||||
virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
|
||||
|
||||
/* Query the mapping outputName=>outputPath for the given derivation.
|
||||
Assume every output has a mapping and throw an exception otherwise. */
|
||||
|
|
@ -468,6 +476,18 @@ public:
|
|||
virtual StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
|
||||
|
||||
/**
|
||||
* Add a mapping indicating that `deriver!outputName` maps to the output path
|
||||
* `output`.
|
||||
*
|
||||
* This is redundant for known-input-addressed and fixed-output derivations
|
||||
* as this information is already present in the drv file, but necessary for
|
||||
* floating-ca derivations and their dependencies as there's no way to
|
||||
* retrieve this information otherwise.
|
||||
*/
|
||||
virtual void registerDrvOutput(const Realisation & output)
|
||||
{ unsupported("registerDrvOutput"); }
|
||||
|
||||
/* Write a NAR dump of a store path. */
|
||||
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#pragma once
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "serialise.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
|
|
@ -50,6 +53,8 @@ typedef enum {
|
|||
wopAddToStoreNar = 39,
|
||||
wopQueryMissing = 40,
|
||||
wopQueryDerivationOutputMap = 41,
|
||||
wopRegisterDrvOutput = 42,
|
||||
wopQueryRealisation = 43,
|
||||
} WorkerOp;
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ namespace nix {
|
|||
typedef enum {
|
||||
lvlError = 0,
|
||||
lvlWarn,
|
||||
lvlNotice,
|
||||
lvlInfo,
|
||||
lvlTalkative,
|
||||
lvlChatty,
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
|
|||
} while (0)
|
||||
|
||||
#define printError(args...) printMsg(lvlError, args)
|
||||
#define notice(args...) printMsg(lvlNotice, args)
|
||||
#define printInfo(args...) printMsg(lvlInfo, args)
|
||||
#define printTalkative(args...) printMsg(lvlTalkative, args)
|
||||
#define debug(args...) printMsg(lvlDebug, args)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ namespace nix {
|
|||
// URI stuff.
|
||||
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
|
||||
const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
|
||||
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
|
||||
const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+";
|
||||
const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
|
||||
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
|
||||
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
|
||||
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
|
||||
|
|
|
|||
|
|
@ -409,7 +409,7 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
|
|||
for (auto & drvInfo : drvInfos) {
|
||||
res.push_back({
|
||||
state->store->parseStorePath(drvInfo.queryDrvPath()),
|
||||
state->store->parseStorePath(drvInfo.queryOutPath()),
|
||||
state->store->maybeParseStorePath(drvInfo.queryOutPath()),
|
||||
drvInfo.queryOutputName()
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -250,7 +250,7 @@ void mainWrapped(int argc, char * * argv)
|
|||
if (legacy) return legacy(argc, argv);
|
||||
}
|
||||
|
||||
verbosity = lvlWarn;
|
||||
verbosity = lvlNotice;
|
||||
settings.verboseBuild = false;
|
||||
evalSettings.pureEval = true;
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
|
|||
};
|
||||
|
||||
if (!json)
|
||||
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
|
||||
notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
|
||||
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
|
||||
|
|
|
|||
|
|
@ -3,3 +3,31 @@ source common.sh
|
|||
file=build-hook.nix
|
||||
|
||||
source build-remote.sh
|
||||
|
||||
# Add a `post-build-hook` option to the nix conf.
|
||||
# This hook will be executed both for the local machine and the remote builders
|
||||
# (because they share the same config).
|
||||
registerBuildHook () {
|
||||
# Dummy post-build-hook just to ensure that it's executed correctly.
|
||||
# (we can't reuse the one from `$PWD/push-to-store.sh` because of
|
||||
# https://github.com/NixOS/nix/issues/4341)
|
||||
cat <<EOF > $TEST_ROOT/post-build-hook.sh
|
||||
#!/bin/sh
|
||||
|
||||
echo "Post hook ran successfully"
|
||||
# Add an empty line to a counter file, just to check that this hook ran properly
|
||||
echo "" >> $TEST_ROOT/post-hook-counter
|
||||
EOF
|
||||
chmod +x $TEST_ROOT/post-build-hook.sh
|
||||
rm -f $TEST_ROOT/post-hook-counter
|
||||
|
||||
echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf
|
||||
}
|
||||
|
||||
registerBuildHook
|
||||
source build-remote.sh
|
||||
|
||||
# `build-hook.nix` has four derivations to build, and the hook runs twice for
|
||||
# each derivation (once on the builder and once on the host), so the counter
|
||||
# should contain eight lines now
|
||||
[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]]
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ builders=(
|
|||
"ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
|
||||
)
|
||||
|
||||
chmod -R +w $TEST_ROOT/machine* || true
|
||||
rm -rf $TEST_ROOT/machine* || true
|
||||
|
||||
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
|
||||
# child process. This allows us to test LegacySSHStore::buildDerivation().
|
||||
# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().
|
||||
|
|
|
|||
|
|
@ -50,7 +50,14 @@ testGC () {
|
|||
nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
|
||||
}
|
||||
|
||||
testRemoteCache
|
||||
testNixCommand () {
|
||||
clearStore
|
||||
nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
|
||||
}
|
||||
|
||||
# Disabled until we have it properly working
|
||||
# testRemoteCache
|
||||
testDeterministicCA
|
||||
testCutoff
|
||||
testGC
|
||||
testNixCommand
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ keep-derivations = false
|
|||
sandbox = false
|
||||
experimental-features = nix-command flakes
|
||||
gc-reserved-space = 0
|
||||
substituters =
|
||||
flake-registry = $TEST_ROOT/registry.json
|
||||
include nix.conf.extra
|
||||
EOF
|
||||
|
|
|
|||
Loading…
Reference in a new issue