Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

This commit is contained in:
John Ericson 2023-04-17 13:54:48 -04:00
commit ab5ca608bf
50 changed files with 1154 additions and 442 deletions

View file

@ -203,10 +203,9 @@ Most Nix commands accept the following command-line options:
instead. instead.
- <span id="opt-I">[`-I`](#opt-I)</span> *path*\ - <span id="opt-I">[`-I`](#opt-I)</span> *path*\
Add a path to the Nix expression search path. This option may be Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path).
given multiple times. See the `NIX_PATH` environment variable for This option may be given multiple times.
information on the semantics of the Nix search path. Paths added Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH).
through `-I` take precedence over `NIX_PATH`.
- <span id="opt-option">[`--option`](#opt-option)</span> *name* *value*\ - <span id="opt-option">[`--option`](#opt-option)</span> *name* *value*\
Set the Nix configuration option *name* to *value*. This overrides Set the Nix configuration option *name* to *value*. This overrides

View file

@ -293,7 +293,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
try { try {
auto h = Hash::parseAny(hash, parseHashType(algo)); auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name); auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -350,8 +350,9 @@ connected:
debug("missing output %s", outputName); debug("missing output %s", outputName);
assert(optResult); assert(optResult);
auto & result = *optResult; auto & result = *optResult;
assert(result.builtOutputs.count(thisOutputId)); auto i = result.builtOutputs.find(outputName);
auto newRealisation = result.builtOutputs.at(thisOutputId); assert(i != result.builtOutputs.end());
auto & newRealisation = i->second;
missingRealisations.insert(newRealisation); missingRealisations.insert(newRealisation);
missingPaths.insert(newRealisation.outPath); missingPaths.insert(newRealisation.outPath);
} }

View file

@ -593,8 +593,8 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs) for (auto & [outputName, realisation] : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath); outputs.emplace(outputName, realisation.outPath);
res.push_back({aux.installable, { res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs }, .path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info, .info = aux.info,

View file

@ -1293,7 +1293,13 @@ drvName, Bindings * attrs, Value & v)
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(method, h, drvName); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", drv.outputs.insert_or_assign("out",
DerivationOutput::CAFixed { DerivationOutput::CAFixed {
@ -2103,7 +2109,13 @@ static void addPath(
std::optional<StorePath> expectedStorePath; std::optional<StorePath> expectedStorePath;
if (expectedHash) if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = *expectedHash,
},
.references = {},
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode StorePath dstPath = settings.readOnlyMode

View file

@ -243,10 +243,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// early exit if pinned and already in the store // early exit if pinned and already in the store
if (expectedHash && expectedHash->type == htSHA256) { if (expectedHash && expectedHash->type == htSHA256) {
auto expectedPath = auto expectedPath = state.store->makeFixedOutputPath(
unpack name,
? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) FixedOutputInfo {
: state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); .hash = {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
},
.references = {}
});
if (state.store->isValidPath(expectedPath)) { if (state.store->isValidPath(expectedPath)) {
state.allowAndSetStorePathString(expectedPath, v); state.allowAndSetStorePathString(expectedPath, v);

View file

@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const
auto narHash = getNarHash(); auto narHash = getNarHash();
if (!narHash) if (!narHash)
throw Error("cannot compute store path for unlocked input '%s'", to_string()); throw Error("cannot compute store path for unlocked input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); return store.makeFixedOutputPath(getName(), FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
.references = {},
});
} }
std::string Input::getType() const std::string Input::getType() const

View file

@ -71,15 +71,19 @@ DownloadFileResult downloadFile(
dumpString(res.data, sink); dumpString(res.data, sink);
auto hash = hashString(htSHA256, res.data); auto hash = hashString(htSHA256, res.data);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), *store,
name,
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Flat,
.hash = hash,
},
.references = {},
},
hashString(htSHA256, sink.s), hashString(htSHA256, sink.s),
}; };
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash { auto source = StringSource { sink.s };
.method = FileIngestionMethod::Flat,
.hash = hash,
};
auto source = StringSource(sink.s);
store->addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }

View file

@ -306,11 +306,22 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
unsupported("addToStoreFromDump"); unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name, references), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = nar.first,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.references = references;
return info; return info;
})->path; })->path;
} }
@ -414,15 +425,22 @@ StorePath BinaryCacheStore::addToStore(
}); });
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, h, name, references), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.references = references;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info; return info;
})->path; })->path;
} }
@ -434,7 +452,7 @@ StorePath BinaryCacheStore::addTextToStore(
RepairFlag repair) RepairFlag repair)
{ {
auto textHash = hashString(htSHA256, s); auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references); auto path = makeTextPath(name, TextInfo { { textHash }, references });
if (!repair && isValidPath(path)) if (!repair && isValidPath(path))
return path; return path;
@ -443,10 +461,16 @@ StorePath BinaryCacheStore::addTextToStore(
dumpString(s, sink); dumpString(s, sink);
StringSource source(sink.s); StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first }; ValidPathInfo info {
*this,
std::string { name },
TextInfo {
{ .hash = textHash },
references,
},
nar.first,
};
info.narSize = nar.second; info.narSize = nar.second;
info.ca = TextHash { textHash };
info.references = references;
return info; return info;
})->path; })->path;
} }

View file

@ -83,16 +83,11 @@ struct BuildResult
*/ */
bool isNonDeterministic = false; bool isNonDeterministic = false;
/**
* The derivation we built or the store path we substituted.
*/
DerivedPath path;
/** /**
* For derivations, a mapping from the names of the wanted outputs * For derivations, a mapping from the names of the wanted outputs
* to actual paths. * to actual paths.
*/ */
DrvOutputs builtOutputs; SingleDrvOutputs builtOutputs;
/** /**
* The start/stop times of the build (or one of the rounds, if it * The start/stop times of the build (or one of the rounds, if it
@ -116,4 +111,15 @@ struct BuildResult
} }
}; };
/**
* A `BuildResult` together with its "primary key".
*/
struct KeyedBuildResult : BuildResult
{
/**
* The derivation we built or the store path we substituted.
*/
DerivedPath path;
};
} }

View file

@ -145,8 +145,20 @@ void DerivationGoal::work()
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{ {
auto newWanted = wantedOutputs.union_(outputs); auto newWanted = wantedOutputs.union_(outputs);
switch (needRestart) {
case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed:
if (!newWanted.isSubsetOf(wantedOutputs)) if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = true; needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
break;
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
/* No need to check whether we added more outputs, because a
restart is already queued up. */
break;
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
/* We are already building all outputs, so it doesn't matter if
we now want more. */
break;
};
wantedOutputs = newWanted; wantedOutputs = newWanted;
} }
@ -297,12 +309,29 @@ void DerivationGoal::outputsSubstitutionTried()
In particular, it may be the case that the hole in the closure is In particular, it may be the case that the hole in the closure is
an output of the current derivation, which causes a loop if retried. an output of the current derivation, which causes a loop if retried.
*/ */
if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; {
bool substitutionFailed =
nrIncompleteClosure > 0 &&
nrIncompleteClosure == nrFailed;
switch (retrySubstitution) {
case RetrySubstitution::NoNeed:
if (substitutionFailed)
retrySubstitution = RetrySubstitution::YesNeed;
break;
case RetrySubstitution::YesNeed:
// Should not be able to reach this state from here.
assert(false);
break;
case RetrySubstitution::AlreadyRetried:
debug("substitution failed again, but we already retried once. Not retrying again.");
break;
}
}
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
if (needRestart) { if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
needRestart = false; needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
haveDerivation(); haveDerivation();
return; return;
} }
@ -330,6 +359,10 @@ void DerivationGoal::outputsSubstitutionTried()
produced using a substitute. So we have to build instead. */ produced using a substitute. So we have to build instead. */
void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::gaveUpOnSubstitution()
{ {
/* At this point we are building all outputs, so if more are wanted there
is no need to restart. */
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
/* The inputs must be built before we can build this goal. */ /* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear(); inputDrvOutputs.clear();
if (useDerivation) if (useDerivation)
@ -451,8 +484,8 @@ void DerivationGoal::inputsRealised()
return; return;
} }
if (retrySubstitution && !retriedSubstitution) { if (retrySubstitution == RetrySubstitution::YesNeed) {
retriedSubstitution = true; retrySubstitution = RetrySubstitution::AlreadyRetried;
haveDerivation(); haveDerivation();
return; return;
} }
@ -570,8 +603,6 @@ void DerivationGoal::inputsRealised()
build hook. */ build hook. */
state = &DerivationGoal::tryToBuild; state = &DerivationGoal::tryToBuild;
worker.wakeUp(shared_from_this()); worker.wakeUp(shared_from_this());
buildResult = BuildResult { .path = buildResult.path };
} }
void DerivationGoal::started() void DerivationGoal::started()
@ -982,7 +1013,7 @@ void DerivationGoal::resolvedFinished()
auto resolvedDrv = *resolvedDrvGoal->drv; auto resolvedDrv = *resolvedDrvGoal->drv;
auto & resolvedResult = resolvedDrvGoal->buildResult; auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs; SingleDrvOutputs builtOutputs;
if (resolvedResult.success()) { if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
@ -1008,7 +1039,7 @@ void DerivationGoal::resolvedFinished()
worker.store.printStorePath(drvPath), wantedOutput); worker.store.printStorePath(drvPath), wantedOutput);
auto realisation = [&]{ auto realisation = [&]{
auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
if (take1) return *take1; if (take1) return *take1;
/* The above `get` should work. But sateful tracking of /* The above `get` should work. But sateful tracking of
@ -1033,7 +1064,7 @@ void DerivationGoal::resolvedFinished()
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
} }
outputPaths.insert(realisation.outPath); outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation); builtOutputs.emplace(wantedOutput, realisation);
} }
runPostBuildHook( runPostBuildHook(
@ -1158,7 +1189,7 @@ HookReply DerivationGoal::tryBuildHook()
} }
DrvOutputs DerivationGoal::registerOutputs() SingleDrvOutputs DerivationGoal::registerOutputs()
{ {
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have as valid (by doing `nix-store --import'). If so we don't have
@ -1320,7 +1351,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
} }
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{ {
if (!drv->type().isPure()) return { false, {} }; if (!drv->type().isPure()) return { false, {} };
@ -1333,7 +1364,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
return static_cast<StringSet>(names); return static_cast<StringSet>(names);
}, },
}, wantedOutputs.raw()); }, wantedOutputs.raw());
DrvOutputs validOutputs; SingleDrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) { for (auto & i : queryPartialDerivationOutputMap()) {
auto initialOutput = get(initialOutputs, i.first); auto initialOutput = get(initialOutputs, i.first);
@ -1376,7 +1407,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
} }
} }
if (info.wanted && info.known && info.known->isValid()) if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
} }
// If we requested all the outputs, we are always fine. // If we requested all the outputs, we are always fine.
@ -1400,7 +1431,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
} }
DrvOutputs DerivationGoal::assertPathValidity() SingleDrvOutputs DerivationGoal::assertPathValidity()
{ {
auto [allValid, validOutputs] = checkPathValidity(); auto [allValid, validOutputs] = checkPathValidity();
if (!allValid) if (!allValid)
@ -1411,7 +1442,7 @@ DrvOutputs DerivationGoal::assertPathValidity()
void DerivationGoal::done( void DerivationGoal::done(
BuildResult::Status status, BuildResult::Status status,
DrvOutputs builtOutputs, SingleDrvOutputs builtOutputs,
std::optional<Error> ex) std::optional<Error> ex)
{ {
buildResult.status = status; buildResult.status = status;
@ -1452,12 +1483,28 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{ {
Goal::waiteeDone(waitee, result); Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success()) if (!useDerivation) return;
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path)) auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
if (!dg) return;
auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
if (outputs == fullDrv.inputDrvs.end()) return;
for (auto & outputName : outputs->second) {
auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = dg->drvPath,
.outputs = OutputsSpec::Names { outputName },
});
if (buildResult.success()) {
auto i = buildResult.builtOutputs.find(outputName);
if (i != buildResult.builtOutputs.end())
inputDrvOutputs.insert_or_assign( inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName }, { dg->drvPath, outputName },
realisation.outPath); i->second.outPath);
}
}
} }
} }

View file

@ -78,22 +78,58 @@ struct DerivationGoal : public Goal
*/ */
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/**
* See `needRestart`; just for that field.
*/
enum struct NeedRestartForMoreOutputs {
/**
* The goal state machine is progressing based on the current value of
* `wantedOutputs. No actions are needed.
*/
OutputsUnmodifedDontNeed,
/**
* `wantedOutputs` has been extended, but the state machine is
* proceeding according to its old value, so we need to restart.
*/
OutputsAddedDoNeed,
/**
* The goal state machine has progressed to the point of doing a build,
* in which case all outputs will be produced, so extensions to
* `wantedOutputs` no longer require a restart.
*/
BuildInProgressWillNotNeed,
};
/** /**
* Whether additional wanted outputs have been added. * Whether additional wanted outputs have been added.
*/ */
bool needRestart = false; NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
/**
* See `retrySubstitution`; just for that field.
*/
enum RetrySubstitution {
/**
* No issues have yet arose, no need to restart.
*/
NoNeed,
/**
* Something failed and there is an incomplete closure. Let's retry
* substituting.
*/
YesNeed,
/**
* We are current or have already retried substitution, and whether or
* not something goes wrong we will not retry again.
*/
AlreadyRetried,
};
/** /**
* Whether to retry substituting the outputs after building the * Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure. * inputs. This is done in case of an incomplete closure.
*/ */
bool retrySubstitution = false; RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed;
/**
* Whether we've retried substitution, in which case we won't try
* again.
*/
bool retriedSubstitution = false;
/** /**
* The derivation stored at drvPath. * The derivation stored at drvPath.
@ -217,7 +253,7 @@ struct DerivationGoal : public Goal
* Check that the derivation outputs all exist and register them * Check that the derivation outputs all exist and register them
* as valid. * as valid.
*/ */
virtual DrvOutputs registerOutputs(); virtual SingleDrvOutputs registerOutputs();
/** /**
* Open a log file and a pipe to it. * Open a log file and a pipe to it.
@ -270,17 +306,17 @@ struct DerivationGoal : public Goal
* Update 'initialOutputs' to determine the current status of the * Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting * outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a * whether all outputs are valid and non-corrupt, and a
* 'DrvOutputs' structure containing the valid and wanted * 'SingleDrvOutputs' structure containing the valid and wanted
* outputs. * outputs.
*/ */
std::pair<bool, DrvOutputs> checkPathValidity(); std::pair<bool, SingleDrvOutputs> checkPathValidity();
/** /**
* Aborts if any output is not valid or corrupt, and otherwise * Aborts if any output is not valid or corrupt, and otherwise
* returns a 'DrvOutputs' structure containing the wanted * returns a 'SingleDrvOutputs' structure containing the wanted
* outputs. * outputs.
*/ */
DrvOutputs assertPathValidity(); SingleDrvOutputs assertPathValidity();
/** /**
* Forcibly kill the child process, if any. * Forcibly kill the child process, if any.
@ -293,7 +329,7 @@ struct DerivationGoal : public Goal
void done( void done(
BuildResult::Status status, BuildResult::Status status,
DrvOutputs builtOutputs = {}, SingleDrvOutputs builtOutputs = {},
std::optional<Error> ex = {}); std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override; void waiteeDone(GoalPtr waitee, ExitCode result) override;

View file

@ -10,16 +10,8 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
Worker worker(*this, evalStore ? *evalStore : *this); Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals; Goals goals;
for (const auto & br : reqs) { for (auto & br : reqs)
std::visit(overloaded { goals.insert(worker.makeGoal(br, buildMode));
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
worker.run(goals); worker.run(goals);
@ -47,7 +39,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
} }
} }
std::vector<BuildResult> Store::buildPathsWithResults( std::vector<KeyedBuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs, const std::vector<DerivedPath> & reqs,
BuildMode buildMode, BuildMode buildMode,
std::shared_ptr<Store> evalStore) std::shared_ptr<Store> evalStore)
@ -55,23 +47,23 @@ std::vector<BuildResult> Store::buildPathsWithResults(
Worker worker(*this, evalStore ? *evalStore : *this); Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals; Goals goals;
for (const auto & br : reqs) { std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { for (const auto & req : reqs) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); auto goal = worker.makeGoal(req, buildMode);
}, goals.insert(goal);
[&](const DerivedPath::Opaque & bo) { state.push_back({req, goal});
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
} }
worker.run(goals); worker.run(goals);
std::vector<BuildResult> results; std::vector<KeyedBuildResult> results;
for (auto & i : goals) for (auto & [req, goalPtr] : state)
results.push_back(i->buildResult); results.emplace_back(KeyedBuildResult {
goalPtr->getBuildResult(req),
/* .path = */ req,
});
return results; return results;
} }
@ -84,15 +76,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
try { try {
worker.run(Goals{goal}); worker.run(Goals{goal});
return goal->buildResult; return goal->getBuildResult(DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All {},
});
} catch (Error & e) { } catch (Error & e) {
return BuildResult { return BuildResult {
.status = BuildResult::MiscFailure, .status = BuildResult::MiscFailure,
.errorMsg = e.msg(), .errorMsg = e.msg(),
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
}; };
}; };
} }

View file

@ -11,6 +11,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
} }
BuildResult Goal::getBuildResult(const DerivedPath & req) {
BuildResult res { buildResult };
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {
auto & bp = *pbp;
/* Because goals are in general shared between derived paths
that share the same derivation, we need to filter their
results to get back just the results we care about.
*/
for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) {
if (bp.outputs.contains(it->first))
++it;
else
it = res.builtOutputs.erase(it);
}
}
return res;
}
void addToWeakGoals(WeakGoals & goals, GoalPtr p) void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{ {
if (goals.find(p) != goals.end()) if (goals.find(p) != goals.end())

View file

@ -81,11 +81,26 @@ struct Goal : public std::enable_shared_from_this<Goal>
*/ */
ExitCode exitCode = ecBusy; ExitCode exitCode = ecBusy;
protected:
/** /**
* Build result. * Build result.
*/ */
BuildResult buildResult; BuildResult buildResult;
public:
/**
* Project a `BuildResult` with just the information that pertains
* to the given request.
*
* In general, goals may be aliased between multiple requests, and
* the stored `BuildResult` has information for the union of all
* requests. We don't want to leak what the other request are for
* sake of both privacy and determinism, and this "safe accessor"
* ensures we don't.
*/
BuildResult getBuildResult(const DerivedPath &);
/** /**
* Exception containing an error message, if any. * Exception containing an error message, if any.
*/ */
@ -93,7 +108,6 @@ struct Goal : public std::enable_shared_from_this<Goal>
Goal(Worker & worker, DerivedPath path) Goal(Worker & worker, DerivedPath path)
: worker(worker) : worker(worker)
, buildResult { .path = std::move(path) }
{ } { }
virtual ~Goal() virtual ~Goal()

View file

@ -1335,7 +1335,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
result.rethrow(); result.rethrow();
} }
std::vector<BuildResult> buildPathsWithResults( std::vector<KeyedBuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths, const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal, BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr) override std::shared_ptr<Store> evalStore = nullptr) override
@ -2174,7 +2174,7 @@ void LocalDerivationGoal::runChild()
} }
DrvOutputs LocalDerivationGoal::registerOutputs() SingleDrvOutputs LocalDerivationGoal::registerOutputs()
{ {
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have as valid (by doing `nix-store --import'). If so we don't have
@ -2395,27 +2395,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
} }
}; };
auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { auto rewriteRefs = [&]() -> StoreReferences {
/* In the CA case, we need the rewritten refs to calculate the /* In the CA case, we need the rewritten refs to calculate the
final path, therefore we look for a *non-rewritten final path, therefore we look for a *non-rewritten
self-reference, and use a bool rather try to solve the self-reference, and use a bool rather try to solve the
computationally intractable fixed point. */ computationally intractable fixed point. */
std::pair<bool, StorePathSet> res { StoreReferences res {
false, .self = false,
{},
}; };
for (auto & r : references) { for (auto & r : references) {
auto name = r.name(); auto name = r.name();
auto origHash = std::string { r.hashPart() }; auto origHash = std::string { r.hashPart() };
if (r == *scratchPath) { if (r == *scratchPath) {
res.first = true; res.self = true;
} else if (auto outputRewrite = get(outputRewrites, origHash)) { } else if (auto outputRewrite = get(outputRewrites, origHash)) {
std::string newRef = *outputRewrite; std::string newRef = *outputRewrite;
newRef += '-'; newRef += '-';
newRef += name; newRef += name;
res.second.insert(StorePath { newRef }); res.others.insert(StorePath { newRef });
} else { } else {
res.second.insert(r); res.others.insert(r);
} }
} }
return res; return res;
@ -2448,18 +2447,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
break; break;
} }
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto refs = rewriteRefs(); ValidPathInfo newInfo0 {
worker.store,
auto finalPath = worker.store.makeFixedOutputPath(
outputHash.method,
got,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
refs.second, FixedOutputInfo {
refs.first); .hash = {
if (*scratchPath != finalPath) { .method = outputHash.method,
.hash = got,
},
.references = rewriteRefs(),
},
Hash::dummy,
};
if (*scratchPath != newInfo0.path) {
// Also rewrite the output path // Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) { auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
dumpPath(actualPath, rsink2); dumpPath(actualPath, rsink2);
rsink2.flush(); rsink2.flush();
}); });
@ -2470,19 +2473,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
} }
HashResult narHashAndSize = hashPath(htSHA256, actualPath); HashResult narHashAndSize = hashPath(htSHA256, actualPath);
ValidPathInfo newInfo0 { newInfo0.narHash = narHashAndSize.first;
finalPath,
narHashAndSize.first,
};
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
newInfo0.references = refs.second;
if (refs.first)
newInfo0.references.insert(newInfo0.path);
assert(newInfo0.ca); assert(newInfo0.ca);
return newInfo0; return newInfo0;
@ -2504,8 +2496,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
auto refs = rewriteRefs(); auto refs = rewriteRefs();
newInfo0.references = refs.second; newInfo0.references = std::move(refs.others);
if (refs.first) if (refs.self)
newInfo0.references.insert(newInfo0.path); newInfo0.references.insert(newInfo0.path);
return newInfo0; return newInfo0;
}, },
@ -2519,7 +2511,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* Check wanted hash */ /* Check wanted hash */
const Hash & wanted = dof.hash.hash; const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = getContentAddressHash(*newInfo0.ca); auto got = newInfo0.ca->getHash();
if (wanted != got) { if (wanted != got) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */
@ -2691,7 +2683,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
means it's safe to link the derivation to the output hash. We must do means it's safe to link the derivation to the output hash. We must do
that for floating CA derivations, which otherwise couldn't be cached, that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */ but it's fine to do in all cases. */
DrvOutputs builtOutputs; SingleDrvOutputs builtOutputs;
for (auto & [outputName, newInfo] : infos) { for (auto & [outputName, newInfo] : infos) {
auto oldinfo = get(initialOutputs, outputName); auto oldinfo = get(initialOutputs, outputName);
@ -2710,7 +2702,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
worker.store.registerDrvOutput(thisRealisation); worker.store.registerDrvOutput(thisRealisation);
} }
if (wantedOutputs.contains(outputName)) if (wantedOutputs.contains(outputName))
builtOutputs.emplace(thisRealisation.id, thisRealisation); builtOutputs.emplace(outputName, thisRealisation);
} }
return builtOutputs; return builtOutputs;

View file

@ -237,7 +237,7 @@ struct LocalDerivationGoal : public DerivationGoal
* Check that the derivation outputs all exist and register them * Check that the derivation outputs all exist and register them
* as valid. * as valid.
*/ */
DrvOutputs registerOutputs() override; SingleDrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override; void signRealisation(Realisation &) override;

View file

@ -95,7 +95,9 @@ void PathSubstitutionGoal::tryNext()
subs.pop_front(); subs.pop_front();
if (ca) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); subPath = sub->makeFixedOutputPathFromCA(
std::string { storePath.name() },
ContentAddressWithReferences::withoutRefs(*ca));
if (sub->storeDir == worker.store.storeDir) if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath); assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) { } else if (sub->storeDir != worker.store.storeDir) {

View file

@ -92,6 +92,7 @@ std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const Sto
return goal; return goal;
} }
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca) std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
{ {
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id]; std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
@ -104,6 +105,20 @@ std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal
return goal; return goal;
} }
GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
},
}, req.raw());
}
template<typename K, typename G> template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap) static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{ {

View file

@ -181,7 +181,7 @@ public:
*/ */
/** /**
* derivation goal * @ref DerivationGoal "derivation goal"
*/ */
private: private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
@ -196,11 +196,19 @@ public:
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/** /**
* substitution goal * @ref SubstitutionGoal "substitution goal"
*/ */
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/**
* Make a goal corresponding to the `DerivedPath`.
*
* It will be a `DerivationGoal` for a `DerivedPath::Built` or
* a `SubstitutionGoal` for a `DerivedPath::Opaque`.
*/
GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal);
/** /**
* Remove a dead goal. * Remove a dead goal.
*/ */

View file

@ -9,7 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type); return makeFileIngestionPrefix(method) + printHashType(hash.type);
} }
std::string makeFileIngestionPrefix(const FileIngestionMethod m) std::string makeFileIngestionPrefix(FileIngestionMethod m)
{ {
switch (m) { switch (m) {
case FileIngestionMethod::Flat: case FileIngestionMethod::Flat:
@ -21,39 +21,35 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
} }
} }
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string ContentAddress::render() const
{
return "fixed:"
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true);
}
std::string renderContentAddress(ContentAddress ca)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash & th) { [](const TextHash & th) {
return "text:" + th.hash.to_string(Base32, true); return "text:"
+ th.hash.to_string(Base32, true);
}, },
[](FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash); return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
} }
}, ca); }, raw);
} }
std::string renderContentAddressMethod(ContentAddressMethod cam) std::string ContentAddressMethod::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHashMethod & th) { [](const TextHashMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(htSHA256);
}, },
[](FixedOutputHashMethod & fshm) { [](const FixedOutputHashMethod & fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
} }
}, cam); }, raw);
} }
/* /**
Parses content address strings up to the hash. * Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
{ {
@ -97,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
} }
ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddress ContentAddress::parse(std::string_view rawCa) {
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
@ -115,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
}); });
}, },
}, caMethod); }, caMethod.raw);
} }
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
{ {
std::string asPrefix = std::string{caMethod} + ":"; std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference // parseContentAddressMethodPrefix takes its argument by reference
@ -126,26 +122,55 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
return parseContentAddressMethodPrefix(asPrefixView); return parseContentAddressMethodPrefix(asPrefixView);
} }
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) std::optional<ContentAddress> ContentAddress::parseOpt(std::string_view rawCaOpt)
{ {
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); return rawCaOpt == ""
? std::nullopt
: std::optional { ContentAddress::parse(rawCaOpt) };
}; };
std::string renderContentAddress(std::optional<ContentAddress> ca) std::string renderContentAddress(std::optional<ContentAddress> ca)
{ {
return ca ? renderContentAddress(*ca) : ""; return ca ? ca->render() : "";
} }
Hash getContentAddressHash(const ContentAddress & ca) const Hash & ContentAddress::getHash() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextHash & th) { [](const TextHash & th) -> auto & {
return th.hash; return th.hash;
}, },
[](const FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) -> auto & {
return fsh.hash; return fsh.hash;
},
}, raw);
} }
}, ca);
bool StoreReferences::empty() const
{
return !self && others.empty();
}
size_t StoreReferences::size() const
{
return (self ? 1 : 0) + others.size();
}
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo {
.hash = h,
.references = {},
};
},
[&](const FixedOutputHash & h) -> ContentAddressWithReferences {
return FixedOutputInfo {
.hash = h,
.references = {},
};
},
}, ca.raw);
} }
} }

View file

@ -3,12 +3,30 @@
#include <variant> #include <variant>
#include "hash.hh" #include "hash.hh"
#include "path.hh"
#include "comparator.hh" #include "comparator.hh"
namespace nix { namespace nix {
/*
* Content addressing method
*/
/* We only have one way to hash text with references, so this is a single-value
type, mainly useful with std::variant.
*/
/** /**
* An enumeration of the ways we can serialize file system objects. * The single way we can serialize "text" file system objects.
*
* Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently.
*/
struct TextHashMethod : std::monostate { };
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/ */
enum struct FileIngestionMethod : uint8_t { enum struct FileIngestionMethod : uint8_t {
/** /**
@ -22,6 +40,53 @@ enum struct FileIngestionMethod : uint8_t {
Recursive = true Recursive = true
}; };
/**
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested.
*/
std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
};
/**
* An enumeration of all the ways we can serialize file system objects.
*
* Just the type of a content address. Combine with the hash itself, and
* we have a `ContentAddress` as defined below. Combine that, in turn,
* with info on references, and we have `ContentAddressWithReferences`,
* as defined further below.
*/
struct ContentAddressMethod
{
typedef std::variant<
TextHashMethod,
FixedOutputHashMethod
> Raw;
Raw raw;
GENERATE_CMP(ContentAddressMethod, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressMethod(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
static ContentAddressMethod parse(std::string_view rawCaMethod);
std::string render() const;
};
/*
* Mini content address
*/
/** /**
* Somewhat obscure, used by \ref Derivation derivations and * Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently. * `builtins.toFile` currently.
@ -36,7 +101,7 @@ struct TextHash {
}; };
/** /**
* For path computed by makeFixedOutputPath. * Used by most store objects that are content-addressed.
*/ */
struct FixedOutputHash { struct FixedOutputHash {
/** /**
@ -65,41 +130,96 @@ struct FixedOutputHash {
* - fixed:<r?>:<ht>:<h>: For paths computed by * - fixed:<r?>:<ht>:<h>: For paths computed by
* Store::makeFixedOutputPath() / Store::addToStore(). * Store::makeFixedOutputPath() / Store::addToStore().
*/ */
struct ContentAddress
{
typedef std::variant< typedef std::variant<
TextHash, TextHash,
FixedOutputHash FixedOutputHash
> ContentAddress; > Raw;
/** Raw raw;
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested. GENERATE_CMP(ContentAddress, me->raw);
*/
std::string makeFileIngestionPrefix(const FileIngestionMethod m); /* The moral equivalent of `using Raw::Raw;` */
ContentAddress(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/** /**
* Compute the content-addressability assertion (ValidPathInfo::ca) for * Compute the content-addressability assertion (ValidPathInfo::ca) for
* paths created by Store::makeFixedOutputPath() / Store::addToStore(). * paths created by Store::makeFixedOutputPath() / Store::addToStore().
*/ */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); std::string render() const;
std::string renderContentAddress(ContentAddress ca); static ContentAddress parse(std::string_view rawCa);
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
const Hash & getHash() const;
};
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
/* /*
We only have one way to hash text with references, so this is single-value * Full content address
type is only useful in std::variant. *
* See the schema for store paths in store-api.cc
*/ */
struct TextHashMethod { };
struct FixedOutputHashMethod { /**
FileIngestionMethod fileIngestionMethod; * A set of references to other store objects.
HashType hashType; *
* References to other store objects are tracked with store paths, self
* references however are tracked with a boolean.
*/
struct StoreReferences {
/**
* References to other store objects
*/
StorePathSet others;
/**
* Reference to this store object
*/
bool self = false;
/**
* @return true iff no references, i.e. others is empty and self is
* false.
*/
bool empty() const;
/**
* Returns the numbers of references, i.e. the size of others + 1
* iff self is true.
*/
size_t size() const;
GENERATE_CMP(StoreReferences, me->self, me->others);
};
// This matches the additional info that we need for makeTextPath
struct TextInfo {
TextHash hash;
/**
* References to other store objects only; self references
* disallowed
*/
StorePathSet references;
GENERATE_CMP(TextInfo, me->hash, me->references);
};
struct FixedOutputInfo {
FixedOutputHash hash;
/**
* References to other store objects or this one.
*/
StoreReferences references;
GENERATE_CMP(FixedOutputInfo, me->hash, me->references);
}; };
/** /**
@ -107,13 +227,27 @@ struct FixedOutputHashMethod {
* *
* A ContentAddress without a Hash. * A ContentAddress without a Hash.
*/ */
struct ContentAddressWithReferences
{
typedef std::variant< typedef std::variant<
TextHashMethod, TextInfo,
FixedOutputHashMethod FixedOutputInfo
> ContentAddressMethod; > Raw;
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); Raw raw;
std::string renderContentAddressMethod(ContentAddressMethod caMethod); GENERATE_CMP(ContentAddressWithReferences, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressWithReferences(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/**
* Create a ContentAddressWithReferences from a mere ContentAddress, by
* assuming no references in all cases.
*/
static ContentAddressWithReferences withoutRefs(const ContentAddress &);
};
} }

View file

@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod &) { [&](const TextHashMethod &) {
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](FixedOutputHashMethod & fohm) { [&](const FixedOutputHashMethod & fohm) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod); }, contentAddressMethod.raw);
}(); }();
logger->stopWork(); logger->stopWork();
@ -637,7 +637,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
} }
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
worker_proto::write(*store, to, res.builtOutputs); DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(*store, to, builtOutputs);
} }
break; break;
} }
@ -880,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
from >> info.registrationTime >> info.narSize >> info.ultimate; from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from); info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from)); info.ca = ContentAddress::parseOpt(readString(from));
from >> repair >> dontCheckSigs; from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs) if (!trusted && dontCheckSigs)
dontCheckSigs = false; dontCheckSigs = false;

View file

@ -36,8 +36,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{ {
return store.makeFixedOutputPath( return store.makeFixedOutputPath(
hash.method, hash.hash, outputPathName(drvName, outputName),
outputPathName(drvName, outputName)); { hash, {} });
} }
@ -942,7 +942,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {

View file

@ -22,6 +22,9 @@
#include <dlfcn.h> #include <dlfcn.h>
#endif #endif
#include "config-impl.hh"
namespace nix { namespace nix {
@ -192,18 +195,18 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
{SandboxMode::smDisabled, false}, {SandboxMode::smDisabled, false},
}); });
template<> void BaseSetting<SandboxMode>::set(const std::string & str, bool append) template<> SandboxMode BaseSetting<SandboxMode>::parse(const std::string & str) const
{ {
if (str == "true") value = smEnabled; if (str == "true") return smEnabled;
else if (str == "relaxed") value = smRelaxed; else if (str == "relaxed") return smRelaxed;
else if (str == "false") value = smDisabled; else if (str == "false") return smDisabled;
else throw UsageError("option '%s' has invalid value '%s'", name, str); else throw UsageError("option '%s' has invalid value '%s'", name, str);
} }
template<> bool BaseSetting<SandboxMode>::isAppendable() template<> struct BaseSetting<SandboxMode>::trait
{ {
return false; static constexpr bool appendable = false;
} };
template<> std::string BaseSetting<SandboxMode>::to_string() const template<> std::string BaseSetting<SandboxMode>::to_string() const
{ {
@ -235,23 +238,23 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
}); });
} }
void MaxBuildJobsSetting::set(const std::string & str, bool append) unsigned int MaxBuildJobsSetting::parse(const std::string & str) const
{ {
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); if (str == "auto") return std::max(1U, std::thread::hardware_concurrency());
else { else {
if (auto n = string2Int<decltype(value)>(str)) if (auto n = string2Int<decltype(value)>(str))
value = *n; return *n;
else else
throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
} }
} }
void PluginFilesSetting::set(const std::string & str, bool append) Paths PluginFilesSetting::parse(const std::string & str) const
{ {
if (pluginsLoaded) if (pluginsLoaded)
throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand");
BaseSetting<Paths>::set(str, append); return BaseSetting<Paths>::parse(str);
} }

View file

@ -26,7 +26,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
options->addSetting(this); options->addSetting(this);
} }
void set(const std::string & str, bool append = false) override; unsigned int parse(const std::string & str) const override;
}; };
struct PluginFilesSetting : public BaseSetting<Paths> struct PluginFilesSetting : public BaseSetting<Paths>
@ -43,7 +43,7 @@ struct PluginFilesSetting : public BaseSetting<Paths>
options->addSetting(this); options->addSetting(this);
} }
void set(const std::string & str, bool append = false) override; Paths parse(const std::string & str) const override;
}; };
const uint32_t maxIdsPerBuild = const uint32_t maxIdsPerBuild =

View file

@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
throw Error("NAR hash is now mandatory"); throw Error("NAR hash is now mandatory");
info->narHash = Hash::parseAnyPrefixed(s); info->narHash = Hash::parseAnyPrefixed(s);
} }
info->ca = parseContentAddressOpt(readString(conn->from)); info->ca = ContentAddress::parseOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from); info->sigs = readStrings<StringSet>(conn->from);
auto s = readString(conn->from); auto s = readString(conn->from);
@ -287,19 +287,18 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult status { BuildResult status;
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
status.status = (BuildResult::Status) readInt(conn->from); status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg; conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
status.builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {}); auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
} }
return status; return status;
} }
@ -330,7 +329,7 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; BuildResult result;
result.status = (BuildResult::Status) readInt(conn->from); result.status = (BuildResult::Status) readInt(conn->from);
if (!result.success()) { if (!result.success()) {

View file

@ -710,6 +710,7 @@ void canonicalisePathMetaData(const Path & path,
canonicalisePathMetaData(path, uidRange, inodesSeen); canonicalisePathMetaData(path, uidRange, inodesSeen);
} }
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{ {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
@ -888,7 +889,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
if (s) info->sigs = tokenizeString<StringSet>(s, " "); if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s); if (s) info->ca = ContentAddress::parseOpt(s);
/* Get the references. */ /* Get the references. */
auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
@ -1221,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
printStorePath(info.path), info.narSize, hashResult.second); printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) { if (info.ca) {
if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) { if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
auto actualFoHash = hashCAPath( auto actualFoHash = hashCAPath(
foHash->method, foHash->method,
foHash->hash.type, foHash->hash.type,
@ -1234,7 +1235,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
actualFoHash.hash.to_string(Base32, true)); actualFoHash.hash.to_string(Base32, true));
} }
} }
if (auto textHash = std::get_if<TextHash>(&*info.ca)) { if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath)); auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) { if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
@ -1320,7 +1321,19 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto [hash, size] = hashSink->finish(); auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name, references); ContentAddressWithReferences desc = FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
};
auto dstPath = makeFixedOutputPathFromCA(name, desc);
addTempRoot(dstPath); addTempRoot(dstPath);
@ -1364,10 +1377,13 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
optimisePath(realPath, repair); optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash.first }; ValidPathInfo info {
*this,
name,
std::move(desc),
narHash.first
};
info.narSize = narHash.second; info.narSize = narHash.second;
info.references = references;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info); registerValidPath(info);
} }
@ -1384,7 +1400,10 @@ StorePath LocalStore::addTextToStore(
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto hash = hashString(htSHA256, s); auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references); auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
});
addTempRoot(dstPath); addTempRoot(dstPath);

View file

@ -27,18 +27,17 @@ std::map<StorePath, StorePath> makeContentAddressed(
StringMap rewrites; StringMap rewrites;
StorePathSet references; StoreReferences refs;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) { for (auto & ref : oldInfo->references) {
if (ref == path) if (ref == path)
hasSelfReference = true; refs.self = true;
else { else {
auto i = remappings.find(ref); auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref; auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths? // FIXME: warn about unremapped paths?
if (replacement != ref) if (replacement != ref)
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
references.insert(std::move(replacement)); refs.others.insert(std::move(replacement));
} }
} }
@ -49,24 +48,28 @@ std::map<StorePath, StorePath> makeContentAddressed(
auto narModuloHash = hashModuloSink.finish().first; auto narModuloHash = hashModuloSink.finish().first;
auto dstPath = dstStore.makeFixedOutputPath( ValidPathInfo info {
FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); dstStore,
path.name(),
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
},
.references = std::move(refs),
},
Hash::dummy,
};
printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
StringSink sink2; StringSink sink2;
RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
rsink2(sink.s); rsink2(sink.s);
rsink2.flush(); rsink2.flush();
ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; info.narHash = hashString(htSHA256, sink2.s);
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
};
StringSource source(sink2.s); StringSource source(sink2.s);
dstStore.addToStore(info, source); dstStore.addToStore(info, source);

View file

@ -273,7 +273,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9)); narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig); narInfo->sigs.insert(sig);
narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11));
return {oValid, narInfo}; return {oValid, narInfo};
}); });

View file

@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "CA") { else if (name == "CA") {
if (ca) throw corrupt(); if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field? // FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value); ca = ContentAddress::parseOpt(value);
} }
pos = eol + 1; pos = eol + 1;

View file

@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
uint64_t fileSize = 0; uint64_t fileSize = 0;
NarInfo() = delete; NarInfo() = delete;
NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash)
: ValidPathInfo(store, std::move(name), std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence); NarInfo(const Store & store, const std::string & s, const std::string & whence);

View file

@ -21,25 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store))); sigs.insert(secretKey.signDetached(fingerprint(store)));
} }
std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const
bool ValidPathInfo::isContentAddressed(const Store & store) const
{ {
if (! ca) return false; if (! ca)
return std::nullopt;
auto caPath = std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & th) { [&](const TextHash & th) -> ContentAddressWithReferences {
return store.makeTextPath(path.name(), th.hash, references); assert(references.count(path) == 0);
return TextInfo {
.hash = th,
.references = references,
};
}, },
[&](const FixedOutputHash & fsh) { [&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
auto refs = references; auto refs = references;
bool hasSelfReference = false; bool hasSelfReference = false;
if (refs.count(path)) { if (refs.count(path)) {
hasSelfReference = true; hasSelfReference = true;
refs.erase(path); refs.erase(path);
} }
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); return FixedOutputInfo {
.hash = foh,
.references = {
.others = std::move(refs),
.self = hasSelfReference,
},
};
},
}, ca->raw);
} }
}, *ca);
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto fullCaOpt = contentAddressWithReferences();
if (! fullCaOpt)
return false;
auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt);
bool res = caPath == path; bool res = caPath == path;
@ -77,6 +97,29 @@ Strings ValidPathInfo::shortRefs() const
} }
ValidPathInfo::ValidPathInfo(
const Store & store,
std::string_view name,
ContentAddressWithReferences && ca,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(name, ca))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = std::move(ti.references);
this->ca = std::move((TextHash &&) ti);
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references.others);
if (foi.references.self)
this->references.insert(path);
this->ca = std::move((FixedOutputHash &&) foi);
},
}, std::move(ca).raw);
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{ {
return read(source, store, format, store.parseStorePath(readString(source))); return read(source, store, format, store.parseStorePath(readString(source)));
@ -93,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
if (format >= 16) { if (format >= 16) {
source >> info.ultimate; source >> info.ultimate;
info.sigs = readStrings<StringSet>(source); info.sigs = readStrings<StringSet>(source);
info.ca = parseContentAddressOpt(readString(source)); info.ca = ContentAddress::parseOpt(readString(source));
} }
return info; return info;
} }

View file

@ -92,6 +92,13 @@ struct ValidPathInfo
void sign(const Store & store, const SecretKey & secretKey); void sign(const Store & store, const SecretKey & secretKey);
/**
* @return The `ContentAddressWithReferences` that determines the
* store path for a content-addressed store object, `std::nullopt`
* for an input-addressed store object.
*/
std::optional<ContentAddressWithReferences> contentAddressWithReferences() const;
/** /**
* @return true iff the path is verifiably content-addressed. * @return true iff the path is verifiably content-addressed.
*/ */
@ -118,6 +125,9 @@ struct ValidPathInfo
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(const Store & store,
std::string_view name, ContentAddressWithReferences && ca, Hash narHash);
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
static ValidPathInfo read(Source & source, const Store & store, unsigned int format); static ValidPathInfo read(Source & source, const Store & store, unsigned int format);

View file

@ -13,9 +13,25 @@ namespace nix {
class Store; class Store;
/**
* A general `Realisation` key.
*
* This is similar to a `DerivedPath::Opaque`, but the derivation is
* identified by its "hash modulo" instead of by its store path.
*/
struct DrvOutput { struct DrvOutput {
// The hash modulo of the derivation /**
* The hash modulo of the derivation.
*
* Computed from the derivation itself for most types of
* derivations, but computed from the (fixed) content address of the
* output for fixed-output derivations.
*/
Hash drvHash; Hash drvHash;
/**
* The name of the output.
*/
std::string outputName; std::string outputName;
std::string to_string() const; std::string to_string() const;
@ -60,6 +76,21 @@ struct Realisation {
GENERATE_CMP(Realisation, me->id, me->outPath); GENERATE_CMP(Realisation, me->id, me->outPath);
}; };
/**
* Collection type for a single derivation's outputs' `Realisation`s.
*
* Since these are the outputs of a single derivation, we know the
* output names are unique so we can use them as the map key.
*/
typedef std::map<std::string, Realisation> SingleDrvOutputs;
/**
* Collection type for multiple derivations' outputs' `Realisation`s.
*
* `DrvOutput` is used because in general the derivations are not all
* the same, so we need to identify firstly which derivation, and
* secondly which output of that derivation.
*/
typedef std::map<DrvOutput, Realisation> DrvOutputs; typedef std::map<DrvOutput, Realisation> DrvOutputs;
struct OpaquePath { struct OpaquePath {

View file

@ -78,7 +78,7 @@ void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & o
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{ {
return parseContentAddress(readString(from)); return ContentAddress::parse(readString(from));
} }
void write(const Store & store, Sink & out, const ContentAddress & ca) void write(const Store & store, Sink & out, const ContentAddress & ca)
@ -125,10 +125,26 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
} }
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _) KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
{ {
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {}); auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
BuildResult res { .path = path }; auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
return KeyedBuildResult {
std::move(br),
/* .path = */ std::move(path),
};
}
void write(const Store & store, Sink & to, const KeyedBuildResult & res)
{
worker_proto::write(store, to, res.path);
worker_proto::write(store, to, static_cast<const BuildResult &>(res));
}
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
{
BuildResult res;
res.status = (BuildResult::Status) readInt(from); res.status = (BuildResult::Status) readInt(from);
from from
>> res.errorMsg >> res.errorMsg
@ -136,13 +152,16 @@ BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
>> res.isNonDeterministic >> res.isNonDeterministic
>> res.startTime >> res.startTime
>> res.stopTime; >> res.stopTime;
res.builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {}); auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
return res; return res;
} }
void write(const Store & store, Sink & to, const BuildResult & res) void write(const Store & store, Sink & to, const BuildResult & res)
{ {
worker_proto::write(store, to, res.path);
to to
<< res.status << res.status
<< res.errorMsg << res.errorMsg
@ -150,7 +169,10 @@ void write(const Store & store, Sink & to, const BuildResult & res)
<< res.isNonDeterministic << res.isNonDeterministic
<< res.startTime << res.startTime
<< res.stopTime; << res.stopTime;
worker_proto::write(store, to, res.builtOutputs); DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(store, to, builtOutputs);
} }
@ -168,7 +190,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
{ {
return parseContentAddressOpt(readString(from)); return ContentAddress::parseOpt(readString(from));
} }
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
@ -586,7 +608,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< renderContentAddressMethod(caMethod); << caMethod.render();
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -644,7 +666,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
} }
} }
}, caMethod); }, caMethod.raw);
auto path = parseStorePath(readString(conn->from)); auto path = parseStorePath(readString(conn->from));
// Release our connection to prevent a deadlock in queryPathInfo(). // Release our connection to prevent a deadlock in queryPathInfo().
conn_.reset(); conn_.reset();
@ -865,7 +887,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
readInt(conn->from); readInt(conn->from);
} }
std::vector<BuildResult> RemoteStore::buildPathsWithResults( std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
const std::vector<DerivedPath> & paths, const std::vector<DerivedPath> & paths,
BuildMode buildMode, BuildMode buildMode,
std::shared_ptr<Store> evalStore) std::shared_ptr<Store> evalStore)
@ -880,7 +902,7 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
writeDerivedPaths(*this, conn, paths); writeDerivedPaths(*this, conn, paths);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::vector<BuildResult>> {}); return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
} else { } else {
// Avoid deadlock. // Avoid deadlock.
conn_.reset(); conn_.reset();
@ -889,21 +911,25 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
// fails, but meh. // fails, but meh.
buildPaths(paths, buildMode, evalStore); buildPaths(paths, buildMode, evalStore);
std::vector<BuildResult> results; std::vector<KeyedBuildResult> results;
for (auto & path : paths) { for (auto & path : paths) {
std::visit( std::visit(
overloaded { overloaded {
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
results.push_back(BuildResult { results.push_back(KeyedBuildResult {
{
.status = BuildResult::Substituted, .status = BuildResult::Substituted,
.path = bo, },
/* .path = */ bo,
}); });
}, },
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
BuildResult res { KeyedBuildResult res {
.status = BuildResult::Built, {
.path = bfd, .status = BuildResult::Built
},
/* .path = */ bfd,
}; };
OutputPathMap outputs; OutputPathMap outputs;
@ -922,10 +948,10 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
queryRealisation(outputId); queryRealisation(outputId);
if (!realisation) if (!realisation)
throw MissingRealisation(outputId); throw MissingRealisation(outputId);
res.builtOutputs.emplace(realisation->id, *realisation); res.builtOutputs.emplace(output, *realisation);
} else { } else {
res.builtOutputs.emplace( res.builtOutputs.emplace(
outputId, output,
Realisation { Realisation {
.id = outputId, .id = outputId,
.outPath = outputPath, .outPath = outputPath,
@ -952,12 +978,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv); writeDerivation(conn->to, *this, drv);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
BuildResult res { BuildResult res;
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
res.status = (BuildResult::Status) readInt(conn->from); res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg; conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
@ -965,7 +986,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
} }
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {}); auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
res.builtOutputs = builtOutputs; for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
} }
return res; return res;
} }

View file

@ -114,7 +114,7 @@ public:
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
std::vector<BuildResult> buildPathsWithResults( std::vector<KeyedBuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths, const std::vector<DerivedPath> & paths,
BuildMode buildMode, BuildMode buildMode,
std::shared_ptr<Store> evalStore) override; std::shared_ptr<Store> evalStore) override;

View file

@ -7,6 +7,7 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "url.hh" #include "url.hh"
#include "references.hh"
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
#include "remote-store.hh" #include "remote-store.hh"
@ -98,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
silly, but it's done that way for compatibility). <id> is the silly, but it's done that way for compatibility). <id> is the
name of the output (usually, "out"). name of the output (usually, "out").
<h2> = base-16 representation of a SHA-256 hash of: <h2> = base-16 representation of a SHA-256 hash of <s2>
<s2> =
if <type> = "text:...": if <type> = "text:...":
the string written to the resulting store path the string written to the resulting store path
if <type> = "source": if <type> = "source:...":
the serialisation of the path from which this store path is the serialisation of the path from which this store path is
copied, as returned by hashPath() copied, as returned by hashPath()
if <type> = "output:<id>": if <type> = "output:<id>":
@ -162,63 +165,63 @@ StorePath Store::makeOutputPath(std::string_view id,
} }
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in, say, <s2> (per the grammar above)
since that would be ambiguous. */
static std::string makeType( static std::string makeType(
const Store & store, const Store & store,
std::string && type, std::string && type,
const StorePathSet & references, const StoreReferences & references)
bool hasSelfReference = false)
{ {
for (auto & i : references) { for (auto & i : references.others) {
type += ":"; type += ":";
type += store.printStorePath(i); type += store.printStorePath(i);
} }
if (hasSelfReference) type += ":self"; if (references.self) type += ":self";
return std::move(type); return std::move(type);
} }
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name);
} else { } else {
assert(references.empty()); assert(info.references.size() == 0);
return makeStorePath("output:out", return makeStorePath("output:out",
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ makeFileIngestionPrefix(method) + makeFileIngestionPrefix(info.hash.method)
+ hash.to_string(Base16, true) + ":"), + info.hash.hash.to_string(Base16, true) + ":"),
name); name);
} }
} }
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
.self = false,
}),
info.hash.hash,
name);
}
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
{ {
// New template // New template
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & th) { [&](const TextInfo & ti) {
return makeTextPath(name, th.hash, references); return makeTextPath(name, ti);
}, },
[&](const FixedOutputHash & fsh) { [&](const FixedOutputInfo & foi) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); return makeFixedOutputPath(name, foi);
} }
}, ca); }, ca.raw);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType(*this, "text", references), hash, name);
} }
@ -228,7 +231,14 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
Hash h = method == FileIngestionMethod::Recursive Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath); : hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h); FixedOutputInfo caInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
} }
@ -237,7 +247,10 @@ StorePath Store::computeStorePathForText(
std::string_view s, std::string_view s,
const StorePathSet & references) const const StorePathSet & references) const
{ {
return makeTextPath(name, hashString(htSHA256, s), references); return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
});
} }
@ -425,11 +438,18 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, hash, name), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {},
},
narHash, narHash,
}; };
info.narSize = narSize; info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = sinkToSource([&](Sink & scratchpadSink) {
@ -521,7 +541,9 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
// Recompute store path so that we can use a different store root. // Recompute store path so that we can use a different store root.
if (path.second) { if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); subPath = makeFixedOutputPathFromCA(
path.first.name(),
ContentAddressWithReferences::withoutRefs(*path.second));
if (sub->storeDir == storeDir) if (sub->storeDir == storeDir)
assert(subPath == path.first); assert(subPath == path.first);
if (subPath != path.first) if (subPath != path.first)
@ -538,10 +560,11 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.first, SubstitutablePathInfo{ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver, .deriver = info->deriver,
info->references, .references = info->references,
narInfo ? narInfo->fileSize : 0, .downloadSize = narInfo ? narInfo->fileSize : 0,
info->narSize}); .narSize = info->narSize,
});
} catch (InvalidPath &) { } catch (InvalidPath &) {
} catch (SubstituterDisabled &) { } catch (SubstituterDisabled &) {
} catch (Error & e) { } catch (Error & e) {
@ -1025,7 +1048,9 @@ void copyStorePath(
// recompute store path on the chance dstStore does it differently // recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info); auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); info2->path = dstStore.makeFixedOutputPathFromCA(
info->path.name(),
info->contentAddressWithReferences().value());
if (dstStore.storeDir == srcStore.storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(info->path == info2->path); assert(info->path == info2->path);
info = info2; info = info2;
@ -1137,7 +1162,9 @@ std::map<StorePath, StorePath> copyPaths(
auto storePathForSrc = currentPathInfo.path; auto storePathForSrc = currentPathInfo.path;
auto storePathForDst = storePathForSrc; auto storePathForDst = storePathForSrc;
if (currentPathInfo.ca && currentPathInfo.references.empty()) { if (currentPathInfo.ca && currentPathInfo.references.empty()) {
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); storePathForDst = dstStore.makeFixedOutputPathFromCA(
currentPathInfo.path.name(),
currentPathInfo.contentAddressWithReferences().value());
if (dstStore.storeDir == srcStore.storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePathForSrc); assert(storePathForDst == storePathForSrc);
if (storePathForDst != storePathForSrc) if (storePathForDst != storePathForSrc)

View file

@ -92,6 +92,7 @@ enum BuildMode { bmNormal, bmRepair, bmCheck };
enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
struct BuildResult; struct BuildResult;
struct KeyedBuildResult;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
@ -268,17 +269,11 @@ public:
StorePath makeOutputPath(std::string_view id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash, StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/** /**
* Preparatory part of addToStore(). * Preparatory part of addToStore().
@ -575,7 +570,7 @@ public:
* case of a build/substitution error, this function won't throw an * case of a build/substitution error, this function won't throw an
* exception, but return a BuildResult containing an error message. * exception, but return a BuildResult containing an error message.
*/ */
virtual std::vector<BuildResult> buildPathsWithResults( virtual std::vector<KeyedBuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths, const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal, BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr); std::shared_ptr<Store> evalStore = nullptr);

View file

@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult); MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(, KeyedBuildResult);
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>); MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);

View file

@ -0,0 +1,71 @@
#pragma once
/**
* @file
*
* Template implementations (as opposed to mere declarations).
*
* One only needs to include this when one is declaring a
* `BaseClass<CustomType>` setting, or as derived class of such an
* instantiation.
*/
#include "config.hh"
namespace nix {
template<> struct BaseSetting<Strings>::trait
{
static constexpr bool appendable = true;
};
template<> struct BaseSetting<StringSet>::trait
{
static constexpr bool appendable = true;
};
template<> struct BaseSetting<StringMap>::trait
{
static constexpr bool appendable = true;
};
template<> struct BaseSetting<std::set<ExperimentalFeature>>::trait
{
static constexpr bool appendable = true;
};
template<typename T>
struct BaseSetting<T>::trait
{
static constexpr bool appendable = false;
};
template<typename T>
bool BaseSetting<T>::isAppendable()
{
return trait::appendable;
}
template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append);
template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append);
template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append);
template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append);
template<typename T>
void BaseSetting<T>::appendOrSet(T && newValue, bool append)
{
static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type");
assert(!append);
value = std::move(newValue);
}
template<typename T>
void BaseSetting<T>::set(const std::string & str, bool append)
{
if (experimentalFeatureSettings.isEnabled(experimentalFeature))
appendOrSet(parse(str), append);
else {
assert(experimentalFeature);
warn("Ignoring setting '%s' because experimental feature '%s' is not enabled",
name,
showExperimentalFeature(*experimentalFeature));
}
}
}

View file

@ -3,6 +3,8 @@
#include "abstract-setting-to-json.hh" #include "abstract-setting-to-json.hh"
#include "experimental-features.hh" #include "experimental-features.hh"
#include "config-impl.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -80,6 +82,8 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridd
void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
unsigned int pos = 0; unsigned int pos = 0;
std::vector<std::pair<std::string, std::string>> parsedContents;
while (pos < contents.size()) { while (pos < contents.size()) {
std::string line; std::string line;
while (pos < contents.size() && contents[pos] != '\n') while (pos < contents.size() && contents[pos] != '\n')
@ -125,8 +129,21 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
auto i = tokens.begin(); auto i = tokens.begin();
advance(i, 2); advance(i, 2);
set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow parsedContents.push_back({
name,
concatStringsSep(" ", Strings(i, tokens.end())),
});
}; };
// First apply experimental-feature related settings
for (auto & [name, value] : parsedContents)
if (name == "experimental-features" || name == "extra-experimental-features")
set(name, value);
// Then apply other settings
for (auto & [name, value] : parsedContents)
if (name != "experimental-features" && name != "extra-experimental-features")
set(name, value);
} }
void AbstractConfig::applyConfigFile(const Path & path) void AbstractConfig::applyConfigFile(const Path & path)
@ -202,12 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
{ {
} }
template<typename T>
bool BaseSetting<T>::isAppendable()
{
return false;
}
template<typename T> template<typename T>
void BaseSetting<T>::convertToArg(Args & args, const std::string & category) void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
{ {
@ -231,9 +242,9 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
}); });
} }
template<> void BaseSetting<std::string>::set(const std::string & str, bool append) template<> std::string BaseSetting<std::string>::parse(const std::string & str) const
{ {
value = str; return str;
} }
template<> std::string BaseSetting<std::string>::to_string() const template<> std::string BaseSetting<std::string>::to_string() const
@ -242,11 +253,11 @@ template<> std::string BaseSetting<std::string>::to_string() const
} }
template<typename T> template<typename T>
void BaseSetting<T>::set(const std::string & str, bool append) T BaseSetting<T>::parse(const std::string & str) const
{ {
static_assert(std::is_integral<T>::value, "Integer required."); static_assert(std::is_integral<T>::value, "Integer required.");
if (auto n = string2Int<T>(str)) if (auto n = string2Int<T>(str))
value = *n; return *n;
else else
throw UsageError("setting '%s' has invalid value '%s'", name, str); throw UsageError("setting '%s' has invalid value '%s'", name, str);
} }
@ -258,12 +269,12 @@ std::string BaseSetting<T>::to_string() const
return std::to_string(value); return std::to_string(value);
} }
template<> void BaseSetting<bool>::set(const std::string & str, bool append) template<> bool BaseSetting<bool>::parse(const std::string & str) const
{ {
if (str == "true" || str == "yes" || str == "1") if (str == "true" || str == "yes" || str == "1")
value = true; return true;
else if (str == "false" || str == "no" || str == "0") else if (str == "false" || str == "no" || str == "0")
value = false; return false;
else else
throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
} }
@ -291,16 +302,15 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
}); });
} }
template<> void BaseSetting<Strings>::set(const std::string & str, bool append) template<> Strings BaseSetting<Strings>::parse(const std::string & str) const
{ {
auto ss = tokenizeString<Strings>(str); return tokenizeString<Strings>(str);
if (!append) value.clear();
for (auto & s : ss) value.push_back(std::move(s));
} }
template<> bool BaseSetting<Strings>::isAppendable() template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append)
{ {
return true; if (!append) value.clear();
for (auto && s : std::move(newValue)) value.push_back(std::move(s));
} }
template<> std::string BaseSetting<Strings>::to_string() const template<> std::string BaseSetting<Strings>::to_string() const
@ -308,16 +318,16 @@ template<> std::string BaseSetting<Strings>::to_string() const
return concatStringsSep(" ", value); return concatStringsSep(" ", value);
} }
template<> void BaseSetting<StringSet>::set(const std::string & str, bool append) template<> StringSet BaseSetting<StringSet>::parse(const std::string & str) const
{ {
if (!append) value.clear(); return tokenizeString<StringSet>(str);
for (auto & s : tokenizeString<StringSet>(str))
value.insert(s);
} }
template<> bool BaseSetting<StringSet>::isAppendable() template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append)
{ {
return true; if (!append) value.clear();
for (auto && s : std::move(newValue))
value.insert(s);
} }
template<> std::string BaseSetting<StringSet>::to_string() const template<> std::string BaseSetting<StringSet>::to_string() const
@ -325,21 +335,24 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value); return concatStringsSep(" ", value);
} }
template<> void BaseSetting<std::set<ExperimentalFeature>>::set(const std::string & str, bool append) template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeature>>::parse(const std::string & str) const
{ {
if (!append) value.clear(); std::set<ExperimentalFeature> res;
for (auto & s : tokenizeString<StringSet>(str)) { for (auto & s : tokenizeString<StringSet>(str)) {
auto thisXpFeature = parseExperimentalFeature(s); auto thisXpFeature = parseExperimentalFeature(s);
if (thisXpFeature) if (thisXpFeature)
value.insert(thisXpFeature.value()); res.insert(thisXpFeature.value());
else else
warn("unknown experimental feature '%s'", s); warn("unknown experimental feature '%s'", s);
} }
return res;
} }
template<> bool BaseSetting<std::set<ExperimentalFeature>>::isAppendable() template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append)
{ {
return true; if (!append) value.clear();
for (auto && s : std::move(newValue))
value.insert(s);
} }
template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
@ -350,20 +363,23 @@ template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() c
return concatStringsSep(" ", stringifiedXpFeatures); return concatStringsSep(" ", stringifiedXpFeatures);
} }
template<> void BaseSetting<StringMap>::set(const std::string & str, bool append) template<> StringMap BaseSetting<StringMap>::parse(const std::string & str) const
{ {
if (!append) value.clear(); StringMap res;
for (auto & s : tokenizeString<Strings>(str)) { for (auto & s : tokenizeString<Strings>(str)) {
auto eq = s.find_first_of('='); auto eq = s.find_first_of('=');
if (std::string::npos != eq) if (std::string::npos != eq)
value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); res.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
// else ignored // else ignored
} }
return res;
} }
template<> bool BaseSetting<StringMap>::isAppendable() template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append)
{ {
return true; if (!append) value.clear();
for (auto && [k, v] : std::move(newValue))
value.emplace(std::move(k), std::move(v));
} }
template<> std::string BaseSetting<StringMap>::to_string() const template<> std::string BaseSetting<StringMap>::to_string() const
@ -387,15 +403,15 @@ template class BaseSetting<StringSet>;
template class BaseSetting<StringMap>; template class BaseSetting<StringMap>;
template class BaseSetting<std::set<ExperimentalFeature>>; template class BaseSetting<std::set<ExperimentalFeature>>;
void PathSetting::set(const std::string & str, bool append) Path PathSetting::parse(const std::string & str) const
{ {
if (str == "") { if (str == "") {
if (allowEmpty) if (allowEmpty)
value = ""; return "";
else else
throw UsageError("setting '%s' cannot be empty", name); throw UsageError("setting '%s' cannot be empty", name);
} else } else
value = canonPath(str); return canonPath(str);
} }
bool GlobalConfig::set(const std::string & name, const std::string & value) bool GlobalConfig::set(const std::string & name, const std::string & value)

View file

@ -215,8 +215,11 @@ protected:
virtual void set(const std::string & value, bool append = false) = 0; virtual void set(const std::string & value, bool append = false) = 0;
virtual bool isAppendable() /**
{ return false; } * Whether the type is appendable; i.e. whether the `append`
* parameter to `set()` is allowed to be `true`.
*/
virtual bool isAppendable() = 0;
virtual std::string to_string() const = 0; virtual std::string to_string() const = 0;
@ -241,6 +244,23 @@ protected:
const T defaultValue; const T defaultValue;
const bool documentDefault; const bool documentDefault;
/**
* Parse the string into a `T`.
*
* Used by `set()`.
*/
virtual T parse(const std::string & str) const;
/**
* Append or overwrite `value` with `newValue`.
*
* Some types to do not support appending in which case `append`
* should never be passed. The default handles this case.
*
* @param append Whether to append or overwrite.
*/
virtual void appendOrSet(T && newValue, bool append);
public: public:
BaseSetting(const T & def, BaseSetting(const T & def,
@ -268,9 +288,25 @@ public:
template<typename U> template<typename U>
void setDefault(const U & v) { if (!overridden) value = v; } void setDefault(const U & v) { if (!overridden) value = v; }
void set(const std::string & str, bool append = false) override; /**
* Require any experimental feature the setting depends on
*
* Uses `parse()` to get the value from `str`, and `appendOrSet()`
* to set it.
*/
void set(const std::string & str, bool append = false) override final;
bool isAppendable() override; /**
* C++ trick; This is template-specialized to compile-time indicate whether
* the type is appendable.
*/
struct trait;
/**
* Always defined based on the C++ magic
* with `trait` above.
*/
bool isAppendable() override final;
virtual void override(const T & v) virtual void override(const T & v)
{ {
@ -336,7 +372,7 @@ public:
options->addSetting(this); options->addSetting(this);
} }
void set(const std::string & str, bool append = false) override; Path parse(const std::string & str) const override;
Path operator +(const char * p) const { return value + p; } Path operator +(const char * p) const { return value + p; }

View file

@ -82,6 +82,7 @@ namespace nix {
TestSetting() : AbstractSetting("test", "test", {}) {} TestSetting() : AbstractSetting("test", "test", {}) {}
void set(const std::string & value, bool append) override {} void set(const std::string & value, bool append) override {}
std::string to_string() const override { return {}; } std::string to_string() const override { return {}; }
bool isAppendable() override { return false; }
}; };
Config config; Config config;
@ -90,6 +91,7 @@ namespace nix {
ASSERT_FALSE(config.set("test", "value")); ASSERT_FALSE(config.set("test", "value"));
config.addSetting(&setting); config.addSetting(&setting);
ASSERT_TRUE(config.set("test", "value")); ASSERT_TRUE(config.set("test", "value"));
ASSERT_FALSE(config.set("extra-test", "value"));
} }
TEST(Config, withInitialValue) { TEST(Config, withInitialValue) {

View file

@ -204,10 +204,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */ /* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs) static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{ {
auto recursive = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
for (auto i : opFlags) for (auto i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive; if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i); else throw UsageError("unknown flag '%1%'", i);
if (opArgs.size() != 3) if (opArgs.size() != 3)
@ -218,7 +218,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
std::string hash = *i++; std::string hash = *i++;
std::string name = *i++; std::string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
.references = {},
})));
} }
@ -935,7 +941,10 @@ static void opServe(Strings opFlags, Strings opArgs)
if (GET_PROTOCOL_MINOR(clientVersion) >= 3) if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { if (GET_PROTOCOL_MINOR(clientVersion) >= 6) {
worker_proto::write(*store, out, status.builtOutputs); DrvOutputs builtOutputs;
for (auto & [output, realisation] : status.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(*store, out, builtOutputs);
} }
break; break;
@ -964,7 +973,7 @@ static void opServe(Strings opFlags, Strings opArgs)
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
in >> info.registrationTime >> info.narSize >> info.ultimate; in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in); info.sigs = readStrings<StringSet>(in);
info.ca = parseContentAddressOpt(readString(in)); info.ca = ContentAddress::parseOpt(readString(in));
if (info.narSize == 0) if (info.narSize == 0)
throw Error("narInfo is too old and missing the narSize field"); throw Error("narInfo is too old and missing the narSize field");

View file

@ -42,14 +42,18 @@ struct CmdAddToStore : MixDryRun, StoreCommand
} }
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(ingestionMethod, hash, *namePart), *store,
std::move(*namePart),
FixedOutputInfo {
.hash = {
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
.references = {},
},
narHash, narHash,
}; };
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = std::optional { FixedOutputHash {
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) { if (!dryRun) {
auto source = StringSource(sink.s); auto source = StringSource(sink.s);

View file

@ -274,11 +274,11 @@ static std::pair<TrustedFlag, std::string> authPeer(const PeerInfo & peer)
* Run a server. The loop opens a socket and accepts new connections from that * Run a server. The loop opens a socket and accepts new connections from that
* socket. * socket.
* *
* @param trustClientOpt If present, force trusting or not trusted the client. * @param forceTrustClientOpt If present, force trusting or not trusted
* Otherwise, decide based on the authentication settings and user credentials * the client. Otherwise, decide based on the authentication settings
* (from the unix domain socket). * and user credentials (from the unix domain socket).
*/ */
static void daemonLoop(std::optional<TrustedFlag> trustClientOpt) static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
{ {
if (chdir("/") == -1) if (chdir("/") == -1)
throw SysError("cannot change current directory"); throw SysError("cannot change current directory");
@ -325,8 +325,8 @@ static void daemonLoop(std::optional<TrustedFlag> trustClientOpt)
TrustedFlag trusted; TrustedFlag trusted;
std::string user; std::string user;
if (trustClientOpt) if (forceTrustClientOpt)
trusted = *trustClientOpt; trusted = *forceTrustClientOpt;
else { else {
peer = getPeerInfo(remote.get()); peer = getPeerInfo(remote.get());
auto [_trusted, _user] = authPeer(peer); auto [_trusted, _user] = authPeer(peer);
@ -436,8 +436,11 @@ static void processStdioConnection(ref<Store> store, TrustedFlag trustClient)
/** /**
* Entry point shared between the new CLI `nix daemon` and old CLI * Entry point shared between the new CLI `nix daemon` and old CLI
* `nix-daemon`. * `nix-daemon`.
*
* @param forceTrustClientOpt See `daemonLoop()` and the parameter with
* the same name over there for details.
*/ */
static void runDaemon(bool stdio, std::optional<TrustedFlag> trustClientOpt) static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt)
{ {
if (stdio) { if (stdio) {
auto store = openUncachedStore(); auto store = openUncachedStore();
@ -445,15 +448,15 @@ static void runDaemon(bool stdio, std::optional<TrustedFlag> trustClientOpt)
// If --force-untrusted is passed, we cannot forward the connection and // If --force-untrusted is passed, we cannot forward the connection and
// must process it ourselves (before delegating to the next store) to // must process it ourselves (before delegating to the next store) to
// force untrusting the client. // force untrusting the client.
if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!trustClientOpt || *trustClientOpt != NotTrusted)) if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted))
forwardStdioConnection(*remoteStore); forwardStdioConnection(*remoteStore);
else else
// `Trusted` is passed in the auto (no override case) because we // `Trusted` is passed in the auto (no override case) because we
// cannot see who is on the other side of a plain pipe. Limiting // cannot see who is on the other side of a plain pipe. Limiting
// access to those is explicitly not `nix-daemon`'s responsibility. // access to those is explicitly not `nix-daemon`'s responsibility.
processStdioConnection(store, trustClientOpt.value_or(Trusted)); processStdioConnection(store, forceTrustClientOpt.value_or(Trusted));
} else } else
daemonLoop(trustClientOpt); daemonLoop(forceTrustClientOpt);
} }
static int main_nix_daemon(int argc, char * * argv) static int main_nix_daemon(int argc, char * * argv)

View file

@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile(
the store. */ the store. */
if (expectedHash) { if (expectedHash) {
hashType = expectedHash->type; hashType = expectedHash->type;
storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
.hash = {
.method = ingestionMethod,
.hash = *expectedHash,
},
.references = {},
});
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = expectedHash; hash = expectedHash;
else else
@ -118,7 +124,7 @@ std::tuple<StorePath, Hash> prefetchFile(
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
storePath = info.path; storePath = info.path;
assert(info.ca); assert(info.ca);
hash = getContentAddressHash(*info.ca); hash = info.ca->getHash();
} }
return {storePath.value(), hash.value()}; return {storePath.value(), hash.value()};

View file

@ -200,12 +200,22 @@ struct ProfileManifest
auto narHash = hashString(htSHA256, sink.s); auto narHash = hashString(htSHA256, sink.s);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), *store,
"profile",
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
.references = {
.others = std::move(references),
// profiles never refer to themselves
.self = false,
},
},
narHash, narHash,
}; };
info.references = std::move(references);
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
StringSource source(sink.s); StringSource source(sink.s);
store->addToStore(info, source); store->addToStore(info, source);

View file

@ -23,20 +23,64 @@ source common.sh
# # Medium case, the configuration effects --help # # Medium case, the configuration effects --help
# grep_both_ways store gc --help # grep_both_ways store gc --help
expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' # Test settings that are gated on experimental features; the setting is ignored
nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' # with a warning if the experimental feature is not enabled. The order of the
# `setting = value` lines in the configuration should not matter.
# 'flakes' experimental-feature is disabled before, ignore and warn
NIX_CONFIG='
experimental-features = nix-command
accept-flake-config = true
' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
grepQuiet "false" $TEST_ROOT/stdout
grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
# 'flakes' experimental-feature is disabled after, ignore and warn
NIX_CONFIG='
accept-flake-config = true
experimental-features = nix-command
' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
grepQuiet "false" $TEST_ROOT/stdout
grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
# 'flakes' experimental-feature is enabled before, process
NIX_CONFIG='
experimental-features = nix-command flakes
accept-flake-config = true
' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
grepQuiet "true" $TEST_ROOT/stdout
grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
# 'flakes' experimental-feature is enabled after, process
NIX_CONFIG='
accept-flake-config = true
experimental-features = nix-command flakes
' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
grepQuiet "true" $TEST_ROOT/stdout
grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
function exit_code_both_ways {
expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null
nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null
# Also, the order should not matter
expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null
nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null
}
exit_code_both_ways show-config --flake-registry 'https://no'
# Double check these are stable # Double check these are stable
nix --experimental-features '' --help nix --experimental-features '' --help 1>/dev/null
nix --experimental-features '' doctor --help nix --experimental-features '' doctor --help 1>/dev/null
nix --experimental-features '' repl --help nix --experimental-features '' repl --help 1>/dev/null
nix --experimental-features '' upgrade-nix --help nix --experimental-features '' upgrade-nix --help 1>/dev/null
# These 3 arguments are currently given to all commands, which is wrong (as not # These 3 arguments are currently given to all commands, which is wrong (as not
# all care). To deal with fixing later, we simply make them require the # all care). To deal with fixing later, we simply make them require the
# nix-command experimental features --- it so happens that the commands we wish # nix-command experimental features --- it so happens that the commands we wish
# stabilizing to do not need them anyways. # stabilizing to do not need them anyways.
for arg in '--print-build-logs' '--offline' '--refresh'; do for arg in '--print-build-logs' '--offline' '--refresh'; do
nix --experimental-features 'nix-command' "$arg" --help nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null
! nix --experimental-features '' "$arg" --help expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null
done done