Merge remote-tracking branch 'origin/master' into cli-guideline

This commit is contained in:
Rok Garbas 2020-12-17 23:38:36 +01:00
commit bdd05a1730
No known key found for this signature in database
GPG key ID: A0E01EF44C27BF00
32 changed files with 604 additions and 180 deletions

View file

@ -1107,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
// Shouldn't happen as the toplevel derivation is not CA. // Shouldn't happen as the toplevel derivation is not CA.
assert(false); assert(false);
}, },
[&](UnknownHashes) { [&](DeferredHash _) {
for (auto & i : outputs) { for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput { DerivationOutput {
@ -1621,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({
static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
{ {
string s = state.forceStringNoCtx(*args[0], pos); string s = state.forceStringNoCtx(*args[0], pos);
parseJSON(state, s, v); try {
parseJSON(state, s, v);
} catch (JSONParseError &e) {
e.addTrace(pos, "while decoding a JSON string");
throw e;
}
} }
static RegisterPrimOp primop_fromJSON({ static RegisterPrimOp primop_fromJSON({

View file

@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
if (!repair && isValidPath(path)) if (!repair && isValidPath(path))
return path; return path;
auto source = StringSource { s }; StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first }; ValidPathInfo info { path, nar.first };
info.narSize = nar.second; info.narSize = nar.second;
@ -443,6 +445,24 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
})->path; })->path;
} }
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
{
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
return {Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
} else {
return std::nullopt;
}
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.toJSON().dump(), "application/json");
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor() ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{ {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);

View file

@ -33,6 +33,9 @@ private:
protected: protected:
// The prefix under which realisation infos will be stored
const std::string realisationsPrefix = "/realisations";
BinaryCacheStore(const Params & params); BinaryCacheStore(const Params & params);
public: public:
@ -99,6 +102,10 @@ public:
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

View file

@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised()
if (useDerivation) { if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
|| fullDrv.type() == DerivationType::DeferredInputAddressed) { ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
/* We are be able to resolve this derivation based on the /* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */ aliasing that resolved derivation goal */
@ -503,9 +504,6 @@ void DerivationGoal::inputsRealised()
Derivation drvResolved { *std::move(attempt) }; Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved); auto pathResolved = writeDerivation(worker.store, drvResolved);
/* Add to memotable to speed up downstream goal's queries with the
original derivation. */
drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'", auto msg = fmt("Resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath), worker.store.printStorePath(drvPath),
@ -2094,6 +2092,16 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
/* Nothing to be done; 'path' must already be valid. */ /* Nothing to be done; 'path' must already be valid. */
} }
void registerDrvOutput(const Realisation & info) override
// XXX: This should probably be allowed as a no-op if the realisation
// corresponds to an allowed derivation
{ throw Error("registerDrvOutput"); }
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
// XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation
{ throw Error("queryRealisation"); }
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
{ {
if (buildMode != bmNormal) throw Error("unsupported build mode"); if (buildMode != bmNormal) throw Error("unsupported build mode");
@ -2872,6 +2880,8 @@ void DerivationGoal::registerOutputs()
for (auto & i : drv->outputsAndOptPaths(worker.store)) { for (auto & i : drv->outputsAndOptPaths(worker.store)) {
if (!i.second.second || !worker.store.isValidPath(*i.second.second)) if (!i.second.second || !worker.store.isValidPath(*i.second.second))
allValid = false; allValid = false;
else
finalOutputs.insert_or_assign(i.first, *i.second.second);
} }
if (allValid) return; if (allValid) return;
} }
@ -3377,21 +3387,14 @@ void DerivationGoal::registerOutputs()
means it's safe to link the derivation to the output hash. We must do means it's safe to link the derivation to the output hash. We must do
that for floating CA derivations, which otherwise couldn't be cached, that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */ but it's fine to do in all cases. */
bool isCaFloating = drv->type() == DerivationType::CAFloating;
auto drvPathResolved = drvPath; if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (!useDerivation && isCaFloating) { auto outputHashes = staticOutputHashes(worker.store, *drv);
/* Once a floating CA derivations reaches this point, it for (auto& [outputName, newInfo] : infos)
must already be resolved, so we don't bother trying to worker.store.registerDrvOutput(Realisation{
downcast drv to get would would just be an empty .id = DrvOutput{outputHashes.at(outputName), outputName},
inputDrvs field. */ .outPath = newInfo.path});
Derivation drv2 { *drv };
drvPathResolved = writeDerivation(worker.store, drv2);
} }
if (useDerivation || isCaFloating)
for (auto & [outputName, newInfo] : infos)
worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
} }

View file

@ -0,0 +1,11 @@
-- Extension of the sql schema for content-addressed derivations.
-- Won't be loaded unless the experimental feature `ca-derivations`
-- is enabled
create table if not exists Realisations (
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade
);

View file

@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopRegisterDrvOutput: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
auto resolvedDrv = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
logger->stopWork();
break;
}
case wopQueryRealisation: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId);
logger->stopWork();
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
break;
}
default: default:
throw Error("invalid operation %1%", op); throw Error("invalid operation %1%", op);
} }

View file

@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
*/ */
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
bool isDeferred = false;
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) { switch (drv.type()) {
case DerivationType::CAFloating:
return UnknownHashes {};
case DerivationType::CAFixed: { case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes; std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
} }
return outputHashes; return outputHashes;
} }
case DerivationType::CAFloating:
isDeferred = true;
break;
case DerivationType::InputAddressed: case DerivationType::InputAddressed:
break; break;
case DerivationType::DeferredInputAddressed: case DerivationType::DeferredInputAddressed:
@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
calls to this function. */ calls to this function. */
std::map<std::string, StringSet> inputs2; std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) { for (auto & i : drv.inputDrvs) {
bool hasUnknownHash = false;
const auto & res = pathDerivationModulo(store, i.first); const auto & res = pathDerivationModulo(store, i.first);
std::visit(overloaded { std::visit(overloaded {
// Regular non-CA derivation, replace derivation // Regular non-CA derivation, replace derivation
[&](Hash drvHash) { [&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
}, },
[&](DeferredHash deferredHash) {
isDeferred = true;
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes // CA derivation's output hashes
[&](CaOutputHashes outputHashes) { [&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" }; std::set<std::string> justOut = { "out" };
@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
justOut); justOut);
} }
}, },
[&](UnknownHashes) {
hasUnknownHash = true;
},
}, res); }, res);
if (hasUnknownHash) {
return UnknownHashes {};
}
} }
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
if (isDeferred)
return DeferredHash { hash };
else
return hash;
}
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
{
std::map<std::string, Hash> res;
std::visit(overloaded {
[&](Hash drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash});
}
},
[&](DeferredHash deferredHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, deferredHash.hash});
}
},
[&](CaOutputHashes outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true));
return res;
} }
@ -719,10 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
} }
std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
Sync<DrvPathResolutions> drvPathResolutions;
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this }; BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation // Input paths that we'll want to rewrite in the derivation
@ -748,4 +771,34 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
return resolved; return resolved;
} }
std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
{
auto drvPath = writeDerivation(store, *this, NoRepair, false);
return Derivation::tryResolve(store, drvPath);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
{
// This is quite dirty and leaky, but will disappear once #4340 is merged
static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
{
auto resolutions = resolutionsCache.lock();
auto resolvedDrvIter = resolutions->find(drvPath);
if (resolvedDrvIter != resolutions->end()) {
auto & [_, resolvedDrv] = *resolvedDrvIter;
return *resolvedDrv;
}
}
/* Try resolve drv and use that path instead. */
auto drv = store.readDerivation(drvPath);
auto attempt = drv.tryResolveUncached(store);
if (!attempt)
return std::nullopt;
/* Store in memo table. */
resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
return *attempt;
}
} }

View file

@ -18,8 +18,6 @@ namespace nix {
/* The traditional non-fixed-output derivation type. */ /* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed struct DerivationOutputInputAddressed
{ {
/* Will need to become `std::optional<StorePath>` once input-addressed
derivations are allowed to depend on cont-addressed derivations */
StorePath path; StorePath path;
}; };
@ -140,10 +138,14 @@ struct Derivation : BasicDerivation
2. Input placeholders are replaced with realized input store paths. */ 2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store); std::optional<BasicDerivation> tryResolve(Store & store);
static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
Derivation() = default; Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
private:
std::optional<BasicDerivation> tryResolveUncached(Store & store);
}; };
@ -174,12 +176,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front. // whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes; typedef std::map<std::string, Hash> CaOutputHashes;
struct UnknownHashes {}; struct DeferredHash { Hash hash; };
typedef std::variant< typedef std::variant<
Hash, // regular DRV normalized hash Hash, // regular DRV normalized hash
CaOutputHashes, // Fixed-output derivation hashes CaOutputHashes, // Fixed-output derivation hashes
UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
> DrvHashModulo; > DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations /* Returns hashes with the details of fixed-output subderivations
@ -207,22 +209,18 @@ typedef std::variant<
*/ */
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
/* Memoisation of hashDerivationModulo(). */ /* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes; typedef std::map<StorePath, DrvHashModulo> DrvHashes;
// FIXME: global, though at least thread-safe. // FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes; extern Sync<DrvHashes> drvHashes;
/* Memoisation of `readDerivation(..).resove()`. */
typedef std::map<
StorePath,
std::optional<StorePath>
> DrvPathResolutions;
// FIXME: global, though at least thread-safe.
// FIXME: arguably overlaps with hashDerivationModulo memo table.
extern Sync<DrvPathResolutions> drvPathResolutions;
bool wantOutput(const string & output, const std::set<string> & wanted); bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source; struct Source;

View file

@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override BuildMode buildMode) override
{ unsupported("buildDerivation"); } { unsupported("buildDerivation"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
}; };
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

View file

@ -333,6 +333,10 @@ public:
auto conn(connections->get()); auto conn(connections->get());
return conn->remoteVersion; return conn->remoteVersion;
} }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
// TODO: Implement
{ unsupported("queryRealisation"); }
}; };
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore; static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;

View file

@ -87,6 +87,7 @@ protected:
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init()
{ {
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + realisationsPrefix);
if (writeDebugInfo) if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo"); createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init(); BinaryCacheStore::init();

View file

@ -42,6 +42,61 @@
namespace nix { namespace nix {
struct LocalStore::State::Stmts {
/* Some precompiled SQLite statements. */
SQLiteStmt RegisterValidPath;
SQLiteStmt UpdatePathInfo;
SQLiteStmt AddReference;
SQLiteStmt QueryPathInfo;
SQLiteStmt QueryReferences;
SQLiteStmt QueryReferrers;
SQLiteStmt InvalidatePath;
SQLiteStmt AddDerivationOutput;
SQLiteStmt RegisterRealisedOutput;
SQLiteStmt QueryValidDerivers;
SQLiteStmt QueryDerivationOutputs;
SQLiteStmt QueryRealisedOutput;
SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths;
};
int getSchema(Path schemaPath)
{
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
if (!string2Int(s, curSchema))
throw Error("'%1%' is corrupt", schemaPath);
}
return curSchema;
}
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
{
const int nixCASchemaVersion = 1;
int curCASchema = getSchema(schemaPath);
if (curCASchema != nixCASchemaVersion) {
if (curCASchema > nixCASchemaVersion) {
throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
curCASchema, nixCASchemaVersion);
}
if (!lockFile(lockFd.get(), ltWrite, false)) {
printInfo("waiting for exclusive access to the Nix store for ca drvs...");
lockFile(lockFd.get(), ltWrite, true);
}
if (curCASchema == 0) {
static const char schema[] =
#include "ca-specific-schema.sql.gen.hh"
;
db.exec(schema);
}
writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
lockFile(lockFd.get(), ltRead, true);
}
}
LocalStore::LocalStore(const Params & params) LocalStore::LocalStore(const Params & params)
: StoreConfig(params) : StoreConfig(params)
@ -60,6 +115,7 @@ LocalStore::LocalStore(const Params & params)
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or(""))) , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
{ {
auto state(_state.lock()); auto state(_state.lock());
state->stmts = std::make_unique<State::Stmts>();
/* Create missing state directories if they don't already exist. */ /* Create missing state directories if they don't already exist. */
createDirs(realStoreDir); createDirs(realStoreDir);
@ -222,32 +278,58 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false); else openDB(*state, false);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
}
/* Prepare SQL statements. */ /* Prepare SQL statements. */
state->stmtRegisterValidPath.create(state->db, state->stmts->RegisterValidPath.create(state->db,
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
state->stmtUpdatePathInfo.create(state->db, state->stmts->UpdatePathInfo.create(state->db,
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
state->stmtAddReference.create(state->db, state->stmts->AddReference.create(state->db,
"insert or replace into Refs (referrer, reference) values (?, ?);"); "insert or replace into Refs (referrer, reference) values (?, ?);");
state->stmtQueryPathInfo.create(state->db, state->stmts->QueryPathInfo.create(state->db,
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
state->stmtQueryReferences.create(state->db, state->stmts->QueryReferences.create(state->db,
"select path from Refs join ValidPaths on reference = id where referrer = ?;"); "select path from Refs join ValidPaths on reference = id where referrer = ?;");
state->stmtQueryReferrers.create(state->db, state->stmts->QueryReferrers.create(state->db,
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
state->stmtInvalidatePath.create(state->db, state->stmts->InvalidatePath.create(state->db,
"delete from ValidPaths where path = ?;"); "delete from ValidPaths where path = ?;");
state->stmtAddDerivationOutput.create(state->db, state->stmts->AddDerivationOutput.create(state->db,
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
state->stmtQueryValidDerivers.create(state->db, state->stmts->QueryValidDerivers.create(state->db,
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
state->stmtQueryDerivationOutputs.create(state->db, state->stmts->QueryDerivationOutputs.create(state->db,
"select id, path from DerivationOutputs where drv = ?;"); "select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to // Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup. // ensure efficient lookup.
state->stmtQueryPathFromHashPart.create(state->db, state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;"); "select path from ValidPaths where path >= ? limit 1;");
state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert or replace into Realisations (drvPath, outputName, outputPath)
values (?, ?, (select id from ValidPaths where path = ?))
;
)");
state->stmts->QueryRealisedOutput.create(state->db,
R"(
select Output.path from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ? and outputName = ?
;
)");
state->stmts->QueryAllRealisedOutputs.create(state->db,
R"(
select outputName, Output.path from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ?
;
)");
}
} }
@ -285,16 +367,7 @@ std::string LocalStore::getUri()
int LocalStore::getSchema() int LocalStore::getSchema()
{ { return nix::getSchema(schemaPath); }
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
if (!string2Int(s, curSchema))
throw Error("'%1%' is corrupt", schemaPath);
}
return curSchema;
}
void LocalStore::openDB(State & state, bool create) void LocalStore::openDB(State & state, bool create)
{ {
@ -581,16 +654,22 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
} }
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) void LocalStore::registerDrvOutput(const Realisation & info)
{ {
auto state(_state.lock()); auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); retrySQLite<void>([&]() {
state->stmts->RegisterRealisedOutput.use()
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
.exec();
});
} }
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
{ {
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use() state.stmts->AddDerivationOutput.use()
(deriver) (deriver)
(outputName) (outputName)
(printStorePath(output)) (printStorePath(output))
@ -607,7 +686,7 @@ uint64_t LocalStore::addValidPath(State & state,
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path)); printStorePath(info.path));
state.stmtRegisterValidPath.use() state.stmts->RegisterValidPath.use()
(printStorePath(info.path)) (printStorePath(info.path))
(info.narHash.to_string(Base16, true)) (info.narHash.to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime) (info.registrationTime == 0 ? time(0) : info.registrationTime)
@ -637,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state,
/* Floating CA derivations have indeterminate output paths until /* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */ they are built, so don't register anything in that case */
if (i.second.second) if (i.second.second)
linkDeriverToPath(state, id, i.first, *i.second.second); cacheDrvOutputMapping(state, id, i.first, *i.second.second);
} }
} }
@ -659,7 +738,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
auto state(_state.lock()); auto state(_state.lock());
/* Get the path info. */ /* Get the path info. */
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
if (!useQueryPathInfo.next()) if (!useQueryPathInfo.next())
return std::shared_ptr<ValidPathInfo>(); return std::shared_ptr<ValidPathInfo>();
@ -679,7 +758,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->registrationTime = useQueryPathInfo.getInt(2); info->registrationTime = useQueryPathInfo.getInt(2);
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
if (s) info->deriver = parseStorePath(s); if (s) info->deriver = parseStorePath(s);
/* Note that narSize = NULL yields 0. */ /* Note that narSize = NULL yields 0. */
@ -687,14 +766,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->ultimate = useQueryPathInfo.getInt(5) == 1; info->ultimate = useQueryPathInfo.getInt(5) == 1;
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
if (s) info->sigs = tokenizeString<StringSet>(s, " "); if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s); if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */ /* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
while (useQueryReferences.next()) while (useQueryReferences.next())
info->references.insert(parseStorePath(useQueryReferences.getStr(0))); info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
@ -709,7 +788,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
/* Update path info in the database. */ /* Update path info in the database. */
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{ {
state.stmtUpdatePathInfo.use() state.stmts->UpdatePathInfo.use()
(info.narSize, info.narSize != 0) (info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true)) (info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate) (info.ultimate ? 1 : 0, info.ultimate)
@ -722,7 +801,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
{ {
auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
if (!use.next()) if (!use.next())
throw InvalidPath("path '%s' is not valid", printStorePath(path)); throw InvalidPath("path '%s' is not valid", printStorePath(path));
return use.getInt(0); return use.getInt(0);
@ -731,7 +810,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
bool LocalStore::isValidPath_(State & state, const StorePath & path) bool LocalStore::isValidPath_(State & state, const StorePath & path)
{ {
return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); return state.stmts->QueryPathInfo.use()(printStorePath(path)).next();
} }
@ -757,7 +836,7 @@ StorePathSet LocalStore::queryAllValidPaths()
{ {
return retrySQLite<StorePathSet>([&]() { return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock()); auto state(_state.lock());
auto use(state->stmtQueryValidPaths.use()); auto use(state->stmts->QueryValidPaths.use());
StorePathSet res; StorePathSet res;
while (use.next()) res.insert(parseStorePath(use.getStr(0))); while (use.next()) res.insert(parseStorePath(use.getStr(0)));
return res; return res;
@ -767,7 +846,7 @@ StorePathSet LocalStore::queryAllValidPaths()
void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
{ {
auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path)));
while (useQueryReferrers.next()) while (useQueryReferrers.next())
referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
@ -788,7 +867,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
return retrySQLite<StorePathSet>([&]() { return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock()); auto state(_state.lock());
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
StorePathSet derivers; StorePathSet derivers;
while (useQueryValidDerivers.next()) while (useQueryValidDerivers.next())
@ -799,69 +878,38 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
} }
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) std::map<std::string, std::optional<StorePath>>
LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
{ {
auto path = path_; auto path = path_;
std::map<std::string, std::optional<StorePath>> outputs; auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
Derivation drv = readDerivation(path);
for (auto & [outName, _] : drv.outputs) {
outputs.insert_or_assign(outName, std::nullopt);
}
bool haveCached = false;
{
auto resolutions = drvPathResolutions.lock();
auto resolvedPathOptIter = resolutions->find(path);
if (resolvedPathOptIter != resolutions->end()) {
auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
if (resolvedPathOpt)
path = *resolvedPathOpt;
haveCached = true;
}
}
/* can't just use else-if instead of `!haveCached` because we need to unlock
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) {
/* Try resolve drv and use that path instead. */
auto attempt = drv.tryResolve(*this);
if (!attempt)
/* If we cannot resolve the derivation, we cannot have any path
assigned so we return the map of all std::nullopts. */
return outputs;
/* Just compute store path */
auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
/* Store in memo table. */
/* FIXME: memo logic should not be local-store specific, should have
wrapper-method instead. */
drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
path = std::move(pathResolved);
}
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock()); auto state(_state.lock());
std::map<std::string, std::optional<StorePath>> outputs;
uint64_t drvId; uint64_t drvId;
try { drvId = queryValidPathId(*state, path);
drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
} catch (InvalidPath &) { while (use.next())
/* FIXME? if the derivation doesn't exist, we cannot have a mapping
for it. */
return outputs;
}
auto useQueryDerivationOutputs {
state->stmtQueryDerivationOutputs.use()
(drvId)
};
while (useQueryDerivationOutputs.next())
outputs.insert_or_assign( outputs.insert_or_assign(
useQueryDerivationOutputs.getStr(0), use.getStr(0), parseStorePath(use.getStr(1)));
parseStorePath(useQueryDerivationOutputs.getStr(1))
);
return outputs; return outputs;
}); });
}
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
return outputs;
auto drv = readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto& [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation)
outputs.insert_or_assign(outputName, realisation->outPath);
else
outputs.insert_or_assign(outputName, std::nullopt);
}
return outputs;
}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
{ {
@ -872,11 +920,11 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> { return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
auto state(_state.lock()); auto state(_state.lock());
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
if (!useQueryPathFromHashPart.next()) return {}; if (!useQueryPathFromHashPart.next()) return {};
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0);
if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
return parseStorePath(s); return parseStorePath(s);
return {}; return {};
@ -990,7 +1038,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
for (auto & [_, i] : infos) { for (auto & [_, i] : infos) {
auto referrer = queryValidPathId(*state, i.path); auto referrer = queryValidPathId(*state, i.path);
for (auto & j : i.references) for (auto & j : i.references)
state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
} }
/* Check that the derivation outputs are correct. We can't do /* Check that the derivation outputs are correct. We can't do
@ -1030,7 +1078,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{ {
debug("invalidating path '%s'", printStorePath(path)); debug("invalidating path '%s'", printStorePath(path));
state.stmtInvalidatePath.use()(printStorePath(path)).exec(); state.stmts->InvalidatePath.use()(printStorePath(path)).exec();
/* Note that the foreign key constraints on the Refs table take /* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */ care of deleting the references entries for `path'. */
@ -1596,5 +1644,18 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
} }
} }
std::optional<const Realisation> LocalStore::queryRealisation(
const DrvOutput& id) {
typedef std::optional<const Realisation> Ret;
return retrySQLite<Ret>([&]() -> Ret {
auto state(_state.lock());
auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
id.outputName));
if (!use.next())
return std::nullopt;
auto outputPath = parseStorePath(use.getStr(0));
return Ret{
Realisation{.id = id, .outPath = outputPath}};
});
} }
} // namespace nix

View file

@ -55,19 +55,8 @@ private:
/* The SQLite database object. */ /* The SQLite database object. */
SQLite db; SQLite db;
/* Some precompiled SQLite statements. */ struct Stmts;
SQLiteStmt stmtRegisterValidPath; std::unique_ptr<Stmts> stmts;
SQLiteStmt stmtUpdatePathInfo;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
/* The file to which we write our temporary roots. */ /* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots; AutoCloseFD fdTempRoots;
@ -138,7 +127,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override; StorePathSet queryValidDerivers(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override; std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@ -219,6 +208,13 @@ public:
garbage until it exceeds maxFree. */ garbage until it exceeds maxFree. */
void autoGC(bool sync = true); void autoGC(bool sync = true);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void registerDrvOutput(const Realisation & info) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
private: private:
int getSchema(); int getSchema();
@ -287,11 +283,6 @@ private:
specified by the secret-key-files option. */ specified by the secret-key-files option. */
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; } Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override; void createUser(const std::string & userName, uid_t userId) override;

View file

@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),)
libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
endif endif
$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(d)/build.cc: $(d)/build.cc:
@ -58,7 +58,7 @@ $(d)/build.cc:
@echo ')foo"' >> $@.tmp @echo ')foo"' >> $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
clean-files += $(d)/schema.sql.gen.hh clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))

View file

@ -0,0 +1,49 @@
#include "realisation.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix {
MakeError(InvalidDerivationOutputId, Error);
DrvOutput DrvOutput::parse(const std::string &strRep) {
size_t n = strRep.find("!");
if (n == strRep.npos)
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
return DrvOutput{
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
.outputName = strRep.substr(n+1),
};
}
std::string DrvOutput::to_string() const {
return strHash() + "!" + outputName;
}
nlohmann::json Realisation::toJSON() const {
return nlohmann::json{
{"id", id.to_string()},
{"outPath", outPath.to_string()},
};
}
Realisation Realisation::fromJSON(
const nlohmann::json& json,
const std::string& whence) {
auto getField = [&](std::string fieldName) -> std::string {
auto fieldIterator = json.find(fieldName);
if (fieldIterator == json.end())
throw Error(
"Drv output info file '%1%' is corrupt, missing field %2%",
whence, fieldName);
return *fieldIterator;
};
return Realisation{
.id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")),
};
}
} // namespace nix

View file

@ -0,0 +1,39 @@
#pragma once
#include "path.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix {
struct DrvOutput {
// The hash modulo of the derivation
Hash drvHash;
std::string outputName;
std::string to_string() const;
std::string strHash() const
{ return drvHash.to_string(Base16, true); }
static DrvOutput parse(const std::string &);
bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
private:
// Just to make comparison operators easier to write
std::pair<Hash, std::string> to_pair() const
{ return std::make_pair(drvHash, outputName); }
};
struct Realisation {
DrvOutput id;
StorePath outPath;
nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
};
typedef std::map<DrvOutput, Realisation> DrvOutputs;
}

View file

@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
} }
void RemoteStore::registerDrvOutput(const Realisation & info)
{
auto conn(getConnection());
conn->to << wopRegisterDrvOutput;
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
conn.processStderr();
}
std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
{
auto conn(getConnection());
conn->to << wopQueryRealisation;
conn->to << id.to_string();
conn.processStderr();
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
if (outPaths.empty())
return std::nullopt;
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
}
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
{ {

View file

@ -81,6 +81,10 @@ public:
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override; void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

View file

@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
dynamic_cast<std::stringstream &>(result.GetBody()).str()); dynamic_cast<std::stringstream &>(result.GetBody()).str());
} catch (S3Error & e) { } catch (S3Error & e) {
if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
(e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw;
} }
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();

View file

@ -10,6 +10,8 @@
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
#include <regex>
namespace nix { namespace nix {
@ -364,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl; return std::chrono::steady_clock::now() < time_point + ttl;
} }
std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
{
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = readInvalidDerivation(path);
for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
outputs.emplace(outputName, output.second);
}
return outputs;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
{
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto resolvedDrv = Derivation::tryResolve(*this, path);
if (resolvedDrv) {
auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
if (isValidPath(resolvedDrvPath))
return queryDerivationOutputMapNoResolve(resolvedDrvPath);
}
}
return queryDerivationOutputMapNoResolve(path);
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path); auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap result; OutputPathMap result;
@ -727,9 +752,17 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
StorePathSet paths2; StorePathSet paths2;
for (auto & path : paths) { for (auto & path : paths) {
if (path.path.isDerivation()) if (path.path.isDerivation()) {
unsupported("buildPaths"); auto outPaths = queryPartialDerivationOutputMap(path.path);
paths2.insert(path.path); for (auto & outputName : path.outputs) {
auto currentOutputPathIter = outPaths.find(outputName);
if (currentOutputPathIter == outPaths.end() ||
!currentOutputPathIter->second ||
!isValidPath(*currentOutputPathIter->second))
unsupported("buildPaths");
}
} else
paths2.insert(path.path);
} }
if (queryValidPaths(paths2).size() != paths2.size()) if (queryValidPaths(paths2).size() != paths2.size())
@ -1091,6 +1124,34 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
} }
} }
// The `parseURL` function supports both IPv6 URIs as defined in
// RFC2732, but also pure addresses. The latter one is needed here to
// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
//
// This function now ensures that a usable connection string is available:
// * If the store to be opened is not an SSH store, nothing will be done.
// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
// needed to pass further flags), it
// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
// * If the URL looks like `root@::1` it will be left as-is.
// * In any other case, the string will be left as-is.
static std::string extractConnStr(const std::string &proto, const std::string &connStr)
{
if (proto.rfind("ssh") != std::string::npos) {
std::smatch result;
std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
if (std::regex_match(connStr, result, v6AddrRegex)) {
if (result[1].matched) {
return result.str(1) + result.str(3);
}
return result.str(3);
}
}
return connStr;
}
ref<Store> openStore(const std::string & uri_, ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams) const Store::Params & extraParams)
{ {
@ -1099,7 +1160,10 @@ ref<Store> openStore(const std::string & uri_,
auto parsedUri = parseURL(uri_); auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end()); params.insert(parsedUri.query.begin(), parsedUri.query.end());
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; auto baseURI = extractConnStr(
parsedUri.scheme,
parsedUri.authority.value_or("") + parsedUri.path
);
for (auto implem : *Implementations::registered) { for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) { if (implem.uriSchemes.count(parsedUri.scheme)) {

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include "realisation.hh"
#include "path.hh" #include "path.hh"
#include "hash.hh" #include "hash.hh"
#include "content-address.hh" #include "content-address.hh"
@ -396,6 +397,8 @@ protected:
public: public:
virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
/* Queries the set of incoming FS references for a store path. /* Queries the set of incoming FS references for a store path.
The result is not cleared. */ The result is not cleared. */
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@ -413,8 +416,13 @@ public:
/* Query the mapping outputName => outputPath for the given derivation. All /* Query the mapping outputName => outputPath for the given derivation. All
outputs are mentioned so ones mising the mapping are mapped to outputs are mentioned so ones mising the mapping are mapped to
`std::nullopt`. */ `std::nullopt`. */
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
{ unsupported("queryPartialDerivationOutputMap"); }
/*
* Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
* the derivation
*/
virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
/* Query the mapping outputName=>outputPath for the given derivation. /* Query the mapping outputName=>outputPath for the given derivation.
Assume every output has a mapping and throw an exception otherwise. */ Assume every output has a mapping and throw an exception otherwise. */
@ -468,6 +476,18 @@ public:
virtual StorePath addTextToStore(const string & name, const string & s, virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0; const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
/**
* Add a mapping indicating that `deriver!outputName` maps to the output path
* `output`.
*
* This is redundant for known-input-addressed and fixed-output derivations
* as this information is already present in the drv file, but necessary for
* floating-ca derivations and their dependencies as there's no way to
* retrieve this information otherwise.
*/
virtual void registerDrvOutput(const Realisation & output)
{ unsupported("registerDrvOutput"); }
/* Write a NAR dump of a store path. */ /* Write a NAR dump of a store path. */
virtual void narFromPath(const StorePath & path, Sink & sink) = 0; virtual void narFromPath(const StorePath & path, Sink & sink) = 0;

View file

@ -1,5 +1,8 @@
#pragma once #pragma once
#include "store-api.hh"
#include "serialise.hh"
namespace nix { namespace nix {
@ -50,6 +53,8 @@ typedef enum {
wopAddToStoreNar = 39, wopAddToStoreNar = 39,
wopQueryMissing = 40, wopQueryMissing = 40,
wopQueryDerivationOutputMap = 41, wopQueryDerivationOutputMap = 41,
wopRegisterDrvOutput = 42,
wopQueryRealisation = 43,
} WorkerOp; } WorkerOp;

View file

@ -45,6 +45,7 @@ namespace nix {
typedef enum { typedef enum {
lvlError = 0, lvlError = 0,
lvlWarn, lvlWarn,
lvlNotice,
lvlInfo, lvlInfo,
lvlTalkative, lvlTalkative,
lvlChatty, lvlChatty,

View file

@ -198,6 +198,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
} while (0) } while (0)
#define printError(args...) printMsg(lvlError, args) #define printError(args...) printMsg(lvlError, args)
#define notice(args...) printMsg(lvlNotice, args)
#define printInfo(args...) printMsg(lvlInfo, args) #define printInfo(args...) printMsg(lvlInfo, args)
#define printTalkative(args...) printMsg(lvlTalkative, args) #define printTalkative(args...) printMsg(lvlTalkative, args)
#define debug(args...) printMsg(lvlDebug, args) #define debug(args...) printMsg(lvlDebug, args)

View file

@ -8,7 +8,8 @@ namespace nix {
// URI stuff. // URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+";
const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";

View file

@ -409,7 +409,7 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
for (auto & drvInfo : drvInfos) { for (auto & drvInfo : drvInfos) {
res.push_back({ res.push_back({
state->store->parseStorePath(drvInfo.queryDrvPath()), state->store->parseStorePath(drvInfo.queryDrvPath()),
state->store->parseStorePath(drvInfo.queryOutPath()), state->store->maybeParseStorePath(drvInfo.queryOutPath()),
drvInfo.queryOutputName() drvInfo.queryOutputName()
}); });
} }

View file

@ -250,7 +250,7 @@ void mainWrapped(int argc, char * * argv)
if (legacy) return legacy(argc, argv); if (legacy) return legacy(argc, argv);
} }
verbosity = lvlWarn; verbosity = lvlNotice;
settings.verboseBuild = false; settings.verboseBuild = false;
evalSettings.pureEval = true; evalSettings.pureEval = true;

View file

@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
}; };
if (!json) if (!json)
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
auto source = sinkToSource([&](Sink & nextSink) { auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);

View file

@ -3,3 +3,31 @@ source common.sh
file=build-hook.nix file=build-hook.nix
source build-remote.sh source build-remote.sh
# Add a `post-build-hook` option to the nix conf.
# This hook will be executed both for the local machine and the remote builders
# (because they share the same config).
registerBuildHook () {
# Dummy post-build-hook just to ensure that it's executed correctly.
# (we can't reuse the one from `$PWD/push-to-store.sh` because of
# https://github.com/NixOS/nix/issues/4341)
cat <<EOF > $TEST_ROOT/post-build-hook.sh
#!/bin/sh
echo "Post hook ran successfully"
# Add an empty line to a counter file, just to check that this hook ran properly
echo "" >> $TEST_ROOT/post-hook-counter
EOF
chmod +x $TEST_ROOT/post-build-hook.sh
rm -f $TEST_ROOT/post-hook-counter
echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf
}
registerBuildHook
source build-remote.sh
# `build-hook.nix` has four derivations to build, and the hook runs twice for
# each derivation (once on the builder and once on the host), so the counter
# should contain eight lines now
[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]]

View file

@ -14,6 +14,9 @@ builders=(
"ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz" "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
) )
chmod -R +w $TEST_ROOT/machine* || true
rm -rf $TEST_ROOT/machine* || true
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
# child process. This allows us to test LegacySSHStore::buildDerivation(). # child process. This allows us to test LegacySSHStore::buildDerivation().
# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().

View file

@ -50,7 +50,14 @@ testGC () {
nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
} }
testRemoteCache testNixCommand () {
clearStore
nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
}
# Disabled until we have it properly working
# testRemoteCache
testDeterministicCA testDeterministicCA
testCutoff testCutoff
testGC testGC
testNixCommand

View file

@ -19,6 +19,7 @@ keep-derivations = false
sandbox = false sandbox = false
experimental-features = nix-command flakes experimental-features = nix-command flakes
gc-reserved-space = 0 gc-reserved-space = 0
substituters =
flake-registry = $TEST_ROOT/registry.json flake-registry = $TEST_ROOT/registry.json
include nix.conf.extra include nix.conf.extra
EOF EOF