Merge remote-tracking branch 'upstream/master' into indexed-store-path-outputs

This commit is contained in:
John Ericson 2022-04-01 03:48:22 +00:00
commit 9c6be01d5f
34 changed files with 752 additions and 378 deletions

1
.gitignore vendored
View file

@ -90,6 +90,7 @@ perl/Makefile.config
/misc/systemd/nix-daemon.service /misc/systemd/nix-daemon.service
/misc/systemd/nix-daemon.socket /misc/systemd/nix-daemon.socket
/misc/systemd/nix-daemon.conf
/misc/upstart/nix-daemon.conf /misc/upstart/nix-daemon.conf
/src/resolve-system-dependencies/resolve-system-dependencies /src/resolve-system-dependencies/resolve-system-dependencies

View file

@ -14,3 +14,21 @@
This function is only available if you enable the experimental This function is only available if you enable the experimental
feature `fetch-closure`. feature `fetch-closure`.
* New experimental feature: *impure derivations*. These are
derivations that can produce a different result every time they're
built. Here is an example:
```nix
stdenv.mkDerivation {
name = "impure";
__impure = true; # marks this derivation as impure
buildCommand = "date > $out";
}
```
Running `nix build` twice on this expression will build the
derivation twice, producing two different content-addressed store
paths. Like fixed-output derivations, impure derivations have access
to the network. Only fixed-output derivations and impure derivations
can depend on an impure derivation.

View file

@ -3,6 +3,7 @@ Description=Nix Daemon
Documentation=man:nix-daemon https://nixos.org/manual Documentation=man:nix-daemon https://nixos.org/manual
RequiresMountsFor=@storedir@ RequiresMountsFor=@storedir@
RequiresMountsFor=@localstatedir@ RequiresMountsFor=@localstatedir@
RequiresMountsFor=@localstatedir@/nix/db
ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
[Service] [Service]

View file

@ -90,7 +90,7 @@ poly_configure_nix_daemon_service() {
ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
_sudo "to run systemd-tmpfiles once to pick that path up" \ _sudo "to run systemd-tmpfiles once to pick that path up" \
sytemd-tmpfiles create --prefix=/nix/var/nix systemd-tmpfiles create --prefix=/nix/var/nix
_sudo "to set up the nix-daemon service" \ _sudo "to set up the nix-daemon service" \
systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"

View file

@ -789,15 +789,50 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
return installables.front(); return installables.front();
} }
BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths) BuiltPaths Installable::build(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode)
{ {
BuiltPaths res; BuiltPaths res;
for (const auto & b : hopefullyBuiltPaths) for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
std::visit( res.push_back(builtPath);
overloaded{ return res;
[&](const DerivedPath::Opaque & bo) { }
res.push_back(BuiltPath::Opaque{bo.path});
}, std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
std::vector<DerivedPath> pathsToBuild;
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
for (auto & i : installables) {
for (auto b : i->toDerivedPaths()) {
pathsToBuild.push_back(b);
backmap[b].push_back(i);
}
}
std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
switch (mode) {
case Realise::Nothing:
case Realise::Derivation:
printMissing(store, pathsToBuild, lvlError);
for (auto & path : pathsToBuild) {
for (auto & installable : backmap[path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
OutputPathMap outputs; OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath); auto drv = evalStore->readDerivation(bfd.drvPath);
@ -809,17 +844,14 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
"the derivation '%s' doesn't have an output named '%s'", "the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output); store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto outputId = DrvOutput outputId { outputHashes.at(output), output };
DrvOutput{outputHashes.at(output), output}; auto realisation = store->queryRealisation(outputId);
auto realisation =
store->queryRealisation(outputId);
if (!realisation) if (!realisation)
throw Error( throw Error(
"cannot operate on an output of unbuilt " "cannot operate on an output of unbuilt "
"content-addressed derivation '%s'", "content-addressed derivation '%s'",
outputId.to_string()); outputId.to_string());
outputs.insert_or_assign( outputs.insert_or_assign(output, realisation->outPath);
output, realisation->outPath);
} else { } else {
// If ca-derivations isn't enabled, assume that // If ca-derivations isn't enabled, assume that
// the output path is statically known. // the output path is statically known.
@ -829,58 +861,45 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
output, *drvOutputs.at(output).second); output, *drvOutputs.at(output).second);
} }
} }
res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
}, },
[&](const DerivedPath::Opaque & bo) {
res.push_back({installable, BuiltPath::Opaque { bo.path }});
}, },
b.raw()); }, path.raw());
}
return res;
} }
BuiltPaths Installable::build( break;
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
auto b = i->toDerivedPaths();
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
}
switch (mode) {
case Realise::Nothing:
case Realise::Derivation:
printMissing(store, pathsToBuild, lvlError);
return getBuiltPaths(evalStore, store, pathsToBuild);
case Realise::Outputs: { case Realise::Outputs: {
BuiltPaths res;
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success()) if (!buildResult.success())
buildResult.rethrow(); buildResult.rethrow();
for (auto & installable : backmap[buildResult.path]) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs) for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath); outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back(BuiltPath::Built { bfd.drvPath, outputs }); res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
res.push_back(BuiltPath::Opaque { bo.path }); res.push_back({installable, BuiltPath::Opaque { bo.path }});
}, },
}, buildResult.path.raw()); }, buildResult.path.raw());
} }
return res;
} }
break;
}
default: default:
assert(false); assert(false);
} }
return res;
} }
BuiltPaths Installable::toBuiltPaths( BuiltPaths Installable::toBuiltPaths(

View file

@ -98,6 +98,13 @@ struct Installable
const std::vector<std::shared_ptr<Installable>> & installables, const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode = bmNormal); BuildMode bMode = bmNormal);
static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode = bmNormal);
static std::set<StorePath> toStorePaths( static std::set<StorePath> toStorePaths(
ref<Store> evalStore, ref<Store> evalStore,
ref<Store> store, ref<Store> store,
@ -185,9 +192,4 @@ ref<eval_cache::EvalCache> openEvalCache(
EvalState & state, EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake); std::shared_ptr<flake::LockedFlake> lockedFlake);
BuiltPaths getBuiltPaths(
ref<Store> evalStore,
ref<Store> store,
const DerivedPaths & hopefullyBuiltPaths);
} }

View file

@ -436,6 +436,7 @@ EvalState::EvalState(
, sBuilder(symbols.create("builder")) , sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args")) , sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed")) , sContentAddressed(symbols.create("__contentAddressed"))
, sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash")) , sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode")) , sOutputHashMode(symbols.create("outputHashMode"))

View file

@ -78,7 +78,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString, sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sContentAddressed, sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode, sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations, sRecurseForDerivations,
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,

View file

@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context; PathSet context;
bool contentAddressed = false; bool contentAddressed = false;
bool isImpure = false;
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::optional<std::string> outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat; std::optional<FileIngestionMethod> ingestionMethod;
StringSet outputs; StringSet outputs;
outputs.insert("out"); outputs.insert("out");
@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
settings.requireExperimentalFeature(Xp::CaDerivations); settings.requireExperimentalFeature(Xp::CaDerivations);
} }
else if (i->name == state.sImpure) {
isImpure = state.forceBool(*i->value, pos);
if (isImpure)
settings.requireExperimentalFeature(Xp::ImpureDerivations);
}
/* The `args' attribute is special: it supplies the /* The `args' attribute is special: it supplies the
command-line arguments to the builder. */ command-line arguments to the builder. */
else if (i->name == state.sArgs) { else if (i->name == state.sArgs) {
@ -1183,27 +1190,43 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
.errPos = posDrvName .errPos = posDrvName
}); });
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256"));
Hash h = newHashAllowEmpty(*outputHash, ht); Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", drv.outputs.insert_or_assign("out",
DerivationOutput::CAFixed { DerivationOutput::CAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = ingestionMethod, .method = method,
.hash = std::move(h), .hash = std::move(h),
}, },
}); });
} }
else if (contentAddressed) { else if (contentAddressed || isImpure) {
HashType ht = parseHashType(outputHashAlgo); if (contentAddressed && isImpure)
throw EvalError({
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
.errPos = posDrvName
});
auto ht = parseHashType(outputHashAlgo.value_or("sha256"));
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
for (auto & i : outputs) { for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i); drv.env[i] = hashPlaceholder(i);
if (isImpure)
drv.outputs.insert_or_assign(i,
DerivationOutput::Impure {
.method = method,
.hashType = ht,
});
else
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating { DerivationOutput::CAFloating {
.method = ingestionMethod, .method = method,
.hashType = ht, .hashType = ht,
}); });
} }
@ -1222,34 +1245,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
DerivationOutput::Deferred { }); DerivationOutput::Deferred { });
} }
// Regular, non-CA derivation should always return a single hash and not auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
// hash per output. switch (hashModulo.kind) {
auto hashModulo = hashDerivationModulo(*state.store, drv, true);
std::visit(overloaded {
[&](const DrvHash & drvHash) {
auto & h = drvHash.hash;
switch (drvHash.kind) {
case DrvHash::Kind::Deferred:
/* Outputs already deferred, nothing to do */
break;
case DrvHash::Kind::Regular: case DrvHash::Kind::Regular:
for (auto & [outputName, output] : drv.outputs) { for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(outputName, h, drvName); auto h = hashModulo.hashes.at(i);
drv.env[outputName] = state.store->printStorePath(outPath); auto outPath = state.store->makeOutputPath(i, h, drvName);
output = DerivationOutput::InputAddressed { drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutputInputAddressed {
.path = std::move(outPath), .path = std::move(outPath),
}; });
} }
break; break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
}
} }
},
[&](const CaOutputHashes &) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
},
hashModulo.raw());
} }
/* Write the resulting term into the Nix store directory. */ /* Write the resulting term into the Nix store directory. */

View file

@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict,
.msg = hintfmt("cannot convert %1% to JSON", showType(v)), .msg = hintfmt("cannot convert %1% to JSON", showType(v)),
.errPos = v.determinePos(pos) .errPos = v.determinePos(pos)
}); });
throw e.addTrace(pos, hintfmt("message for the trace")); e.addTrace(pos, hintfmt("message for the trace"));
throw e;
} }
} }

View file

@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
{ {
trace("have derivation"); trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (!drv->type().hasKnownOutputPaths()) if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations); settings.requireExperimentalFeature(Xp::CaDerivations);
if (!drv->type().isPure()) {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
assert(!worker.store.isValidPath(randomPath));
initialOutputs.insert({
outputName,
InitialOutput {
.wanted = true,
.outputHash = impureOutputHash,
.known = InitialOutputStatus {
.path = randomPath,
.status = PathStatus::Absent
}
}
});
}
gaveUpOnSubstitution();
return;
}
for (auto & i : drv->outputsAndOptPaths(worker.store)) for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second) if (i.second.second)
worker.store.addTempRoot(*i.second.second); worker.store.addTempRoot(*i.second.second);
@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
return; return;
} }
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
/* We are first going to try to create the invalid output paths /* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build through substitutes. If that doesn't work, we'll build
them. */ them. */
@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
{ {
trace("all outputs substituted (maybe)"); trace("all outputs substituted (maybe)");
assert(drv->type().isPure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {}, done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::gaveUpOnSubstitution()
{ {
/* The inputs must be built before we can build this goal. */ /* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation) if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (drv->type().isPure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(i.first));
}
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
}
/* Copy the input sources from the eval store to the build /* Copy the input sources from the eval store to the build
store. */ store. */
@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure() void DerivationGoal::repairClosure()
{ {
assert(drv->type().isPure());
/* If we're repairing, we now know that our own outputs are valid. /* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations good. If not, then start derivation goals for the derivations
@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
drvs. */ drvs. */
: true); : true);
}, },
[&](const DerivationType::Impure &) {
return true;
}
}, drvType.raw()); }, drvType.raw());
if (resolveDrv) if (resolveDrv && !fullDrv.inputDrvs.empty()) {
{
settings.requireExperimentalFeature(Xp::CaDerivations); settings.requireExperimentalFeature(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the /* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal now-known results of dependencies. If so, we become a
aliasing that resolved derivation goal */ stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store); std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
assert(attempt); assert(attempt);
Derivation drvResolved { *std::move(attempt) }; Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved); auto pathResolved = writeDerivation(worker.store, drvResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'", auto msg = fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath), worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved)); worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@ -488,23 +528,15 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation /* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths `i' as input paths. Only add the closures of output paths
that are specified as inputs. */ that are specified as inputs. */
assert(worker.evalStore.isValidPath(drvPath)); for (auto & j : wantedDepOutputs)
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
for (auto & j : wantedDepOutputs) { worker.store.computeFSClosure(*outPath, inputPaths);
if (outputs.count(j) > 0) { else
auto optRealizedInput = outputs.at(j);
if (!optRealizedInput)
throw Error(
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
} else
throw Error( throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'", "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
} }
} }
}
/* Second, the input sources. */ /* Second, the input sources. */
worker.store.computeFSClosure(drv->inputSrcs, inputPaths); worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
st = st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected : statusOk(status) ? BuildResult::OutputRejected :
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure : !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure; BuildResult::PermanentFailure;
} }
@ -934,9 +966,15 @@ void DerivationGoal::buildDone()
void DerivationGoal::resolvedFinished() void DerivationGoal::resolvedFinished()
{ {
trace("resolved derivation finished");
assert(resolvedDrvGoal); assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv; auto resolvedDrv = *resolvedDrvGoal->drv;
auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs;
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths; StorePathSet outputPaths;
@ -946,30 +984,22 @@ void DerivationGoal::resolvedFinished()
if (realWantedOutputs.empty()) if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames(); realWantedOutputs = resolvedDrv.outputNames();
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) { for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0); assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation( auto realisation = resolvedResult.builtOutputs.at(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
); if (drv->type().isPure()) {
// We've just built it, but maybe the build failed, in which case the auto newRealisation = realisation;
// realisation won't be there
if (realisation) {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
newRealisation.signatures.clear(); newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
signRealisation(newRealisation); signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
assert(!buildResult.success());
} }
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
} }
runPostBuildHook( runPostBuildHook(
@ -978,16 +1008,11 @@ void DerivationGoal::resolvedFinished()
drvPath, drvPath,
outputPaths outputPaths
); );
auto status = [&]() {
auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) {
case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid;
default:
return resolvedResult.status;
} }
}();
auto status = resolvedResult.status;
if (status == BuildResult::AlreadyValid)
status = BuildResult::ResolvesToAlreadyValid;
done(status, std::move(builtOutputs)); done(status, std::move(builtOutputs));
} }
@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{ {
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) { if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res; std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs) for (auto & [name, output] : drv->outputs)
@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap() OutputPathMap DerivationGoal::queryDerivationOutputMap()
{ {
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) { if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res; OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{ {
if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair; bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs; auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs; DrvOutputs validOutputs;
@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (info.wanted && info.known && info.known->isValid()) if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
} }
// If we requested all the outputs via the empty set, we are always fine. // If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid // If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid. // ones, so any that are left must be invalid.
@ -1343,7 +1373,6 @@ void DerivationGoal::done(
if (ex) if (ex)
// FIXME: strip: "error: " // FIXME: strip: "error: "
buildResult.errorMsg = ex->what(); buildResult.errorMsg = ex->what();
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut) if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true; worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure) if (buildResult.status == BuildResult::PermanentFailure)
@ -1370,7 +1399,21 @@ void DerivationGoal::done(
fs.open(traceBuiltOutputsFile, std::fstream::out); fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
} }
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
} }
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success())
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName },
realisation.outPath);
}
} }

View file

@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
them. */ them. */
StringSet wantedOutputs; StringSet wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */ /* Whether additional wanted outputs have been added. */
bool needRestart = false; bool needRestart = false;
@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
DrvOutputs builtOutputs = {}, DrvOutputs builtOutputs = {},
std::optional<Error> ex = {}); std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths); StorePathSet exportReferences(const StorePathSet & storePaths);
}; };

View file

@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled) else if (settings.sandboxMode == smDisabled)
useChroot = false; useChroot = false;
else if (settings.sandboxMode == smRelaxed) else if (settings.sandboxMode == smRelaxed)
useChroot = !(derivationType.isImpure()) && !noChroot; useChroot = derivationType.isSandboxed() && !noChroot;
} }
auto & localStore = getLocalStore(); auto & localStore = getLocalStore();
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid())); "nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */ /* Create /etc/hosts with localhost entry. */
if (!(derivationType.isImpure())) if (derivationType.isSandboxed())
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot, /* Make the closure of the inputs available in the chroot,
@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us. us.
*/ */
if (!(derivationType.isImpure())) if (derivationType.isSandboxed())
privateNetwork = true; privateNetwork = true;
userNamespaceSync.create(); userNamespaceSync.create();
@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */ already know the cryptographic hash of the output). */
if (derivationType.isImpure()) { if (!derivationType.isSandboxed()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or(""); env[i] = getEnv(i).value_or("");
} }
@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the /* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so network, so give them access to /etc/resolv.conf and so
on. */ on. */
if (derivationType.isImpure()) { if (!derivationType.isSandboxed()) {
// Only use nss functions to resolve hosts and // Only use nss functions to resolve hosts and
// services. Dont use it for anything else that may // services. Dont use it for anything else that may
// be configured for this system. This limits the // be configured for this system. This limits the
@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
if (derivationType.isImpure()) if (!derivationType.isSandboxed())
sandboxProfile += "(import \"sandbox-network.sb\")\n"; sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Add the output paths we'll use at build-time to the chroot */ /* Add the output paths we'll use at build-time to the chroot */
@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
assert(false); assert(false);
}, },
[&](const DerivationOutput::Impure & doi) {
return newInfoFromCA(DerivationOutput::CAFloating {
.method = doi.method,
.hashType = doi.hashType,
});
},
}, output.raw()); }, output.raw());
/* FIXME: set proper permissions in restorePath() so /* FIXME: set proper permissions in restorePath() so
@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
}, },
.outPath = newInfo.path .outPath = newInfo.path
}; };
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
&& drv->type().isPure())
{
signRealisation(thisRealisation); signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation); worker.store.registerDrvOutput(thisRealisation);
} }

View file

@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> { [](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt; return std::nullopt;
}, },
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
return std::nullopt;
},
}, raw()); }, raw());
} }
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return store.makeFixedOutputPath( return store.makeFixedOutputPath(
hash.method, hash.hash, hash.method, hash.hash,
outputPathName(drvName, outputName)); outputPathName(drvName, outputName));
} }
bool DerivationType::isCA() const { bool DerivationType::isCA() const
{
/* Normally we do the full `std::visit` to make sure we have /* Normally we do the full `std::visit` to make sure we have
exhaustively handled all variants, but so long as there is a exhaustively handled all variants, but so long as there is a
variant called `ContentAddressed`, it must be the only one for variant called `ContentAddressed`, it must be the only one for
which `isCA` is true for this to make sense!. */ which `isCA` is true for this to make sense!. */
return std::holds_alternative<ContentAddressed>(raw()); return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return true;
},
}, raw());
} }
bool DerivationType::isFixed() const { bool DerivationType::isFixed() const
{
return std::visit(overloaded { return std::visit(overloaded {
[](const InputAddressed & ia) { [](const InputAddressed & ia) {
return false; return false;
@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
[](const ContentAddressed & ca) { [](const ContentAddressed & ca) {
return ca.fixed; return ca.fixed;
}, },
[](const Impure &) {
return false;
},
}, raw()); }, raw());
} }
bool DerivationType::hasKnownOutputPaths() const { bool DerivationType::hasKnownOutputPaths() const
{
return std::visit(overloaded { return std::visit(overloaded {
[](const InputAddressed & ia) { [](const InputAddressed & ia) {
return !ia.deferred; return !ia.deferred;
@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
[](const ContentAddressed & ca) { [](const ContentAddressed & ca) {
return ca.fixed; return ca.fixed;
}, },
[](const Impure &) {
return false;
},
}, raw()); }, raw());
} }
bool DerivationType::isImpure() const { bool DerivationType::isSandboxed() const
{
return std::visit(overloaded { return std::visit(overloaded {
[](const InputAddressed & ia) { [](const InputAddressed & ia) {
return false; return true;
}, },
[](const ContentAddressed & ca) { [](const ContentAddressed & ca) {
return !ca.pure; return ca.sandboxed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isPure() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return true;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return false;
}, },
}, raw()); }, raw());
} }
@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
hashAlgo = hashAlgo.substr(2); hashAlgo = hashAlgo.substr(2);
} }
const auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash != "") { if (hash == "impure") {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
assert(pathS == "");
return DerivationOutput::Impure {
.method = std::move(method),
.hashType = std::move(hashType),
};
} else if (hash != "") {
validatePath(pathS); validatePath(pathS);
return DerivationOutput::CAFixed { return DerivationOutput::CAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutputImpure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure");
} }
}, i.second.raw()); }, i.second.raw());
s += ')'; s += ')';
@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const DerivationType BasicDerivation::type() const
{ {
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; std::set<std::string_view>
inputAddressedOutputs,
fixedCAOutputs,
floatingCAOutputs,
deferredIAOutputs,
impureOutputs;
std::optional<HashType> floatingHashType; std::optional<HashType> floatingHashType;
for (auto & i : outputs) { for (auto & i : outputs) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivationOutput::InputAddressed &) { [&](const DerivationOutput::InputAddressed &) {
@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
floatingHashType = dof.hashType; floatingHashType = dof.hashType;
} else { } else {
if (*floatingHashType != dof.hashType) if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type"); throw Error("all floating outputs must use the same hash type");
} }
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first); deferredIAOutputs.insert(i.first);
}, },
[&](const DerivationOutput::Impure &) {
impureOutputs.insert(i.first);
},
}, i.second.raw()); }, i.second.raw());
} }
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { if (inputAddressedOutputs.empty()
throw Error("Must have at least one output"); && fixedCAOutputs.empty()
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { && floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
throw Error("must have at least one output");
if (!inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed { return DerivationType::InputAddressed {
.deferred = false, .deferred = false,
}; };
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& !fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
{
if (fixedCAOutputs.size() > 1) if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature? // FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now"); throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out") if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\""); throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed { return DerivationType::ContentAddressed {
.pure = false, .sandboxed = false,
.fixed = true, .fixed = true,
}; };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { }
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& !floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::ContentAddressed { return DerivationType::ContentAddressed {
.pure = true, .sandboxed = true,
.fixed = false, .fixed = false,
}; };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& !deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed { return DerivationType::InputAddressed {
.deferred = true, .deferred = true,
}; };
} else {
throw Error("Can't mix derivation output types"); if (inputAddressedOutputs.empty()
} && fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& !impureOutputs.empty())
return DerivationType::Impure { };
throw Error("can't mix derivation output types");
} }
@ -474,7 +571,7 @@ Sync<DrvHashes> drvHashes;
/* Look up the derivation by value and memoize the /* Look up the derivation by value and memoize the
`hashDerivationModulo` call. `hashDerivationModulo` call.
*/ */
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
{ {
{ {
auto hashes = drvHashes.lock(); auto hashes = drvHashes.lock();
@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
don't leak the provenance of fixed outputs, reducing pointless cache don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this. misses as the build itself won't know this.
*/ */
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
auto type = drv.type(); auto type = drv.type();
@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
return outputHashes; return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
}
if (!type.isPure()) {
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs)
outputHashes.insert_or_assign(outputName, impureOutputHash);
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Deferred,
};
} }
auto kind = std::visit(overloaded { auto kind = std::visit(overloaded {
@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
? DrvHash::Kind::Regular ? DrvHash::Kind::Regular
: DrvHash::Kind::Deferred; : DrvHash::Kind::Deferred;
}, },
[](const DerivationType::Impure &) -> DrvHash::Kind {
assert(false);
}
}, drv.type().raw()); }, drv.type().raw());
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2; std::map<std::string, StringSet> inputs2;
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
// Avoid lambda capture restriction with standard / Clang // Avoid lambda capture restriction with standard / Clang
auto & inputOutputs = inputOutputs0; auto & inputOutputs = inputOutputs0;
const auto & res = pathDerivationModulo(store, drvPath); const auto & res = pathDerivationModulo(store, drvPath);
std::visit(overloaded { if (res.kind == DrvHash::Kind::Deferred)
// Regular non-CA derivation, replace derivation kind = DrvHash::Kind::Deferred;
[&](const DrvHash & drvHash) { for (auto & outputName : inputOutputs) {
kind |= drvHash.kind; const auto h = res.hashes.at(outputName);
inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs); inputs2[h.to_string(Base16, false)].insert(outputName);
},
// CA derivation's output hashes
[&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" };
for (auto & output : inputOutputs) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
} }
},
}, res.raw());
} }
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
return DrvHash { .hash = hash, .kind = kind }; std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs) {
outputHashes.insert_or_assign(outputName, hash);
} }
return DrvHash {
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept .hashes = outputHashes,
{ .kind = kind,
switch (other) { };
case DrvHash::Kind::Regular:
break;
case DrvHash::Kind::Deferred:
self = other;
break;
}
} }
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv) std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
{ {
std::map<std::string, Hash> res; return hashDerivationModulo(store, drv, true).hashes;
std::visit(overloaded {
[&](const DrvHash & drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash.hash});
}
},
[&](const CaOutputHashes & outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true).raw());
return res;
} }
@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
return names; return names;
} }
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
{
DerivationOutputsAndOptPaths outsAndOptPaths; DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs) for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair( outsAndOptPaths.insert(std::make_pair(
@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
return outsAndOptPaths; return outsAndOptPaths;
} }
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
{
auto nameWithSuffix = drvPath.name(); auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv"; constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension)); assert(hasSuffix(nameWithSuffix, extension));
@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< "" << ""
<< ""; << "";
}, },
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
<< "impure";
},
}, i.second.raw()); }, i.second.raw());
} }
worker_proto::write(store, out, drv.inputSrcs); worker_proto::write(store, out, drv.inputSrcs);
@ -723,10 +814,8 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
} }
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
{
debug("Rewriting the derivation");
for (auto & rewrite : rewrites) { for (auto & rewrite : rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second); debug("rewriting %s as %s", rewrite.first, rewrite.second);
} }
@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) { for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) { if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
auto & h = hashModulo.requireNoFixedNonDeferred(); auto & h = hashModulo.hashes.at(outputName);
auto outPath = store.makeOutputPath(outputName, h, drv.name); auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath); drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput::InputAddressed { output = DerivationOutput::InputAddressed {
@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
} }
const Hash & DrvHashModulo::requireNoFixedNonDeferred() const { std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
auto * drvHashOpt = std::get_if<DrvHash>(&raw());
assert(drvHashOpt);
assert(drvHashOpt->kind == DrvHash::Kind::Regular);
return drvHashOpt->hash;
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const StorePath & inputDrv, const StringSet & inputOutputs)
{ {
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv); std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
auto getOutput = [&](const std::string & outputName) { for (auto & input : inputDrvs)
auto & actualPathOpt = inputDrvOutputs.at(outputName); for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
if (!actualPathOpt) if (outputPath)
warn("output %s of input %s missing, aborting the resolving", inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
for (auto & outputName : inputOutputs) { return tryResolve(store, inputDrvOutputs);
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(actualPath));
inputSrcs.insert(std::move(actualPath));
} }
return true; std::optional<BasicDerivation> Derivation::tryResolve(
} Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { {
BasicDerivation resolved { *this }; BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation // Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites; StringMap inputRewrites;
for (auto & [inputDrv, inputOutputs] : inputDrvs) for (auto & [inputDrv, inputOutputs] : inputDrvs) {
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs)) for (auto & outputName : inputOutputs) {
return std::nullopt; if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {
warn("output '%s' of input '%s' missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv));
return {};
}
}
}
rewriteDerivation(store, resolved, inputRewrites); rewriteDerivation(store, resolved, inputRewrites);
return resolved; return resolved;
} }
const Hash impureOutputHash = hashString(htSHA256, "impure");
} }

View file

@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
}; };
/* Input-addressed output which depends on a (CA) derivation whose hash isn't /* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm * known yet.
*/ */
struct DerivationOutputDeferred {}; struct DerivationOutputDeferred {};
/* Impure output which is moved to a content-addressed location (like
CAFloating) but isn't registered as a realization.
*/
struct DerivationOutputImpure
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
typedef std::variant< typedef std::variant<
DerivationOutputInputAddressed, DerivationOutputInputAddressed,
DerivationOutputCAFixed, DerivationOutputCAFixed,
DerivationOutputCAFloating, DerivationOutputCAFloating,
DerivationOutputDeferred DerivationOutputDeferred,
DerivationOutputImpure
> _DerivationOutputRaw; > _DerivationOutputRaw;
struct DerivationOutput : _DerivationOutputRaw struct DerivationOutput : _DerivationOutputRaw
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
using CAFixed = DerivationOutputCAFixed; using CAFixed = DerivationOutputCAFixed;
using CAFloating = DerivationOutputCAFloating; using CAFloating = DerivationOutputCAFloating;
using Deferred = DerivationOutputDeferred; using Deferred = DerivationOutputDeferred;
using Impure = DerivationOutputImpure;
/* Note, when you use this function you should make sure that you're passing /* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer the right derivation name. When in doubt, you should use the safer
@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
}; };
struct DerivationType_ContentAddressed { struct DerivationType_ContentAddressed {
bool pure; bool sandboxed;
bool fixed; bool fixed;
}; };
struct DerivationType_Impure {
};
typedef std::variant< typedef std::variant<
DerivationType_InputAddressed, DerivationType_InputAddressed,
DerivationType_ContentAddressed DerivationType_ContentAddressed,
DerivationType_Impure
> _DerivationTypeRaw; > _DerivationTypeRaw;
struct DerivationType : _DerivationTypeRaw { struct DerivationType : _DerivationTypeRaw {
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
using Raw::Raw; using Raw::Raw;
using InputAddressed = DerivationType_InputAddressed; using InputAddressed = DerivationType_InputAddressed;
using ContentAddressed = DerivationType_ContentAddressed; using ContentAddressed = DerivationType_ContentAddressed;
using Impure = DerivationType_Impure;
/* Do the outputs of the derivation have paths calculated from their content, /* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */ or from the derivation itself? */
@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
non-CA derivations. */ non-CA derivations. */
bool isFixed() const; bool isFixed() const;
/* Is the derivation impure and needs to access non-deterministic resources, or /* Whether the derivation is fully sandboxed. If false, the
pure and can be sandboxed? Note that whether or not we actually sandbox the sandbox is opened up, e.g. the derivation has access to the
derivation is controlled separately. Never true for non-CA derivations. */ network. Note that whether or not we actually sandbox the
bool isImpure() const; derivation is controlled separately. Always true for non-CA
derivations. */
bool isSandboxed() const;
/* Whether the derivation is expected to produce the same result
every time, and therefore it only needs to be built once. This
is only false for derivations that have the attribute '__impure
= true'. */
bool isPure() const;
/* Does the derivation knows its own output paths? /* Does the derivation knows its own output paths?
Only true when there's no floating-ca derivation involved in the Only true when there's no floating-ca derivation involved in the
@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
added directly to input sources. added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */ 2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store); std::optional<BasicDerivation> tryResolve(Store & store) const;
/* Like the above, but instead of querying the Nix database for
realisations, uses a given mapping from input derivation paths
+ output names to actual output store paths. */
std::optional<BasicDerivation> tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
Derivation() = default; Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@ -202,12 +233,14 @@ bool isDerivation(const std::string & fileName);
the output name is "out". */ the output name is "out". */
std::string outputPathName(std::string_view drvName, std::string_view outputName); std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
// The hashes modulo of a derivation.
//
// Each output is given a hash, although in practice only the content-addressed
// derivations (fixed-output or not) will have a different hash for each
// output.
struct DrvHash { struct DrvHash {
Hash hash; std::map<std::string, Hash> hashes;
enum struct Kind : bool { enum struct Kind : bool {
// Statically determined derivations. // Statically determined derivations.
@ -222,28 +255,6 @@ struct DrvHash {
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
typedef std::variant<
// Regular normalized derivation hash, and whether it was deferred (because
// an ancestor derivation is a floating content addressed derivation).
DrvHash,
// Fixed-output derivation hashes
CaOutputHashes
> _DrvHashModuloRaw;
struct DrvHashModulo : _DrvHashModuloRaw {
using Raw = _DrvHashModuloRaw;
using Raw::Raw;
/* Get hash, throwing if it is per-output CA hashes or a
deferred Drv hash.
*/
const Hash & requireNoFixedNonDeferred() const;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
};
/* Returns hashes with the details of fixed-output subderivations /* Returns hashes with the details of fixed-output subderivations
expunged. expunged.
@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw {
ATerm, after subderivations have been likewise expunged from that ATerm, after subderivations have been likewise expunged from that
derivation. derivation.
*/ */
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/* /*
Return a map associating each output to a hash that uniquely identifies its Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references). derivation (modulo the self-references).
FIXME: what is the Hash in this map?
*/ */
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
/* Memoisation of hashDerivationModulo(). */ /* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes; typedef std::map<StorePath, DrvHash> DrvHashes;
// FIXME: global, though at least thread-safe. // FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes; extern Sync<DrvHashes> drvHashes;
@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
dependency which is a CA derivation. */ dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
} }

View file

@ -25,6 +25,9 @@ struct DerivedPathOpaque {
nlohmann::json toJSON(ref<Store> store) const; nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const; std::string to_string(const Store & store) const;
static DerivedPathOpaque parse(const Store & store, std::string_view); static DerivedPathOpaque parse(const Store & store, std::string_view);
bool operator < (const DerivedPathOpaque & b) const
{ return path < b.path; }
}; };
/** /**
@ -46,6 +49,9 @@ struct DerivedPathBuilt {
std::string to_string(const Store & store) const; std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view); static DerivedPathBuilt parse(const Store & store, std::string_view);
nlohmann::json toJSON(ref<Store> store) const; nlohmann::json toJSON(ref<Store> store) const;
bool operator < (const DerivedPathBuilt & b) const
{ return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); }
}; };
using _DerivedPathRaw = std::variant< using _DerivedPathRaw = std::variant<

View file

@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
// combinations that are currently prohibited. // combinations that are currently prohibited.
drv.type(); drv.type();
std::optional<Hash> h; std::optional<DrvHash> hashesModulo;
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doia) { [&](const DerivationOutput::InputAddressed & doia) {
if (!h) { if (!hashesModulo) {
// somewhat expensive so we do lazily // somewhat expensive so we do lazily
auto h0 = hashDerivationModulo(*this, drv, true); hashesModulo = hashDerivationModulo(*this, drv, true);
h = h0.requireNoFixedNonDeferred();
} }
StorePath recomputed = makeOutputPath(i.first, *h, drvName); StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
if (doia.path != recomputed) if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'", throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
/* Nothing to check */ /* Nothing to check */
}, },
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw()); }, i.second.raw());
} }
} }

View file

@ -285,7 +285,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
DrvOutput{outputHashes.at(outputName), outputName}); DrvOutput{outputHashes.at(outputName), outputName});
if (!thisRealisation) if (!thisRealisation)
throw Error( throw Error(
"output '%s' of derivation '%s' isnt built", outputName, "output '%s' of derivation '%s' isn't built", outputName,
store.printStorePath(inputDrv)); store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation); inputRealisations.insert(*thisRealisation);
} }
@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
} }
} }

View file

@ -1,5 +1,7 @@
#include "store-api.hh" #include "store-api.hh"
#include <sodium.h>
namespace nix { namespace nix {
static void checkName(std::string_view path, std::string_view name) static void checkName(std::string_view path, std::string_view name)
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath StorePath::random(std::string_view name)
{
Hash hash(htSHA1);
randombytes_buf(hash.hash, hash.hashSize);
return StorePath(hash, name);
}
StorePath Store::parseStorePath(std::string_view path) const StorePath Store::parseStorePath(std::string_view path) const
{ {
auto p = canonPath(std::string(path)); auto p = canonPath(std::string(path));

View file

@ -58,6 +58,8 @@ public:
} }
static StorePath dummy; static StorePath dummy;
static StorePath random(std::string_view name);
}; };
typedef std::set<StorePath> StorePathSet; typedef std::set<StorePath> StorePathSet;

View file

@ -9,10 +9,9 @@ namespace nix {
const std::string nativeSystem = SYSTEM; const std::string nativeSystem = SYSTEM;
BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
{ {
err.traces.push_front(Trace { .pos = e, .hint = hint }); err.traces.push_front(Trace { .pos = e, .hint = hint });
return *this;
} }
// c++ std::exception descendants must have a 'const char* what()' function. // c++ std::exception descendants must have a 'const char* what()' function.

View file

@ -175,12 +175,12 @@ public:
const ErrorInfo & info() const { calcWhat(); return err; } const ErrorInfo & info() const { calcWhat(); return err; }
template<typename... Args> template<typename... Args>
BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
{ {
return addTrace(e, hintfmt(fs, args...)); addTrace(e, hintfmt(fs, args...));
} }
BaseError & addTrace(std::optional<ErrPos> e, hintformat hint); void addTrace(std::optional<ErrPos> e, hintformat hint);
bool hasTrace() const { return !err.traces.empty(); } bool hasTrace() const { return !err.traces.empty(); }
}; };

View file

@ -7,6 +7,7 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" }, { Xp::CaDerivations, "ca-derivations" },
{ Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" }, { Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" }, { Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" }, { Xp::RecursiveNix, "recursive-nix" },

View file

@ -16,6 +16,7 @@ namespace nix {
enum struct ExperimentalFeature enum struct ExperimentalFeature
{ {
CaDerivations, CaDerivations,
ImpureDerivations,
Flakes, Flakes,
NixCommand, NixCommand,
RecursiveNix, RecursiveNix,

View file

@ -71,14 +71,12 @@ void clearEnv()
unsetenv(name.first.c_str()); unsetenv(name.first.c_str());
} }
void replaceEnv(std::map<std::string, std::string> newEnv) void replaceEnv(const std::map<std::string, std::string> & newEnv)
{ {
clearEnv(); clearEnv();
for (auto newEnvVar : newEnv) for (auto & newEnvVar : newEnv)
{
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
} }
}
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks) Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)

View file

@ -9,7 +9,7 @@ using namespace nix;
struct CmdBundle : InstallableCommand struct CmdBundle : InstallableCommand
{ {
std::string bundler = "github:matthewbauer/nix-bundle"; std::string bundler = "github:NixOS/bundlers";
std::optional<Path> outLink; std::optional<Path> outLink;
CmdBundle() CmdBundle()

View file

@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
output.second = DerivationOutput::Deferred { }; output.second = DerivationOutput::Deferred { };
drv.env[output.first] = ""; drv.env[output.first] = "";
} }
auto h0 = hashDerivationModulo(*evalStore, drv, true); auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
const Hash & h = h0.requireNoFixedNonDeferred();
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
Hash h = hashesModulo.hashes.at(output.first);
auto outPath = store->makeOutputPath(output.first, h, drv.name); auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = DerivationOutput::InputAddressed { output.second = DerivationOutput::InputAddressed {
.path = outPath, .path = outPath,

View file

@ -62,22 +62,21 @@ struct ProfileElement
return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
} }
void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable) void updateStorePaths(
ref<Store> evalStore,
ref<Store> store,
const BuiltPaths & builtPaths)
{ {
// FIXME: respect meta.outputsToInstall // FIXME: respect meta.outputsToInstall
storePaths.clear(); storePaths.clear();
for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) { for (auto & buildable : builtPaths) {
std::visit(overloaded { std::visit(overloaded {
[&](const BuiltPath::Opaque & bo) { [&](const BuiltPath::Opaque & bo) {
storePaths.insert(bo.path); storePaths.insert(bo.path);
}, },
[&](const BuiltPath::Built & bfd) { [&](const BuiltPath::Built & bfd) {
// TODO: Why are we querying if we know the output for (auto & output : bfd.outputs)
// names already? Is it just to figure out what the
// default one is?
for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
storePaths.insert(output.second); storePaths.insert(output.second);
}
}, },
}, buildable.raw()); }, buildable.raw());
} }
@ -236,6 +235,16 @@ struct ProfileManifest
} }
}; };
static std::map<Installable *, BuiltPaths>
builtPathsPerInstallable(
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
{
std::map<Installable *, BuiltPaths> res;
for (auto & [installable, builtPath] : builtPaths)
res[installable.get()].push_back(builtPath);
return res;
}
struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
{ {
std::string description() override std::string description() override
@ -254,7 +263,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
{ {
ProfileManifest manifest(*getEvalState(), *profile); ProfileManifest manifest(*getEvalState(), *profile);
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); auto builtPaths = builtPathsPerInstallable(
Installable::build2(
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
for (auto & installable : installables) { for (auto & installable : installables) {
ProfileElement element; ProfileElement element;
@ -269,7 +280,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
}; };
} }
element.updateStorePaths(getEvalStore(), store, *installable); element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
manifest.elements.push_back(std::move(element)); manifest.elements.push_back(std::move(element));
} }
@ -457,12 +468,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
warn ("Use 'nix profile list' to see the current profile."); warn ("Use 'nix profile list' to see the current profile.");
} }
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); auto builtPaths = builtPathsPerInstallable(
Installable::build2(
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
for (size_t i = 0; i < installables.size(); ++i) { for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i); auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)]; auto & element = manifest.elements[indices.at(i)];
element.updateStorePaths(getEvalStore(), store, *installable); element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
} }
updateProfile(manifest.build(store)); updateProfile(manifest.build(store));

View file

@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded program unshare(CLONE_NEWUSER) doesn't work in a multithreaded program
(which "nix" is), so we exec() a single-threaded helper program (which "nix" is), so we exec() a single-threaded helper program
(chrootHelper() below) to do the work. */ (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>(); auto store2 = store.dynamic_pointer_cast<LocalFSStore>();
if (store2 && store->storeDir != store2->getRealStoreDir()) { if (!store2)
throw Error("store '%s' is not a local store so it does not support command execution", store->getUri());
if (store->storeDir != store2->getRealStoreDir()) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
for (auto & arg : args) helperArgs.push_back(arg); for (auto & arg : args) helperArgs.push_back(arg);

View file

@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
}, },
[&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
outputObj.attr("impure", true);
},
}, output.raw()); }, output.raw());
} }
} }

View file

@ -1,6 +1,6 @@
source common.sh source common.sh
touch foo -t 202211111111 touch $TEST_ROOT/foo -t 202211111111
# We only check whether 2022-11-1* **:**:** is the last modified date since # We only check whether 2022-11-1* **:**:** is the last modified date since
# `lastModified` is transformed into UTC in `builtins.fetchTarball`. # `lastModified` is transformed into UTC in `builtins.fetchTarball`.
[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]] [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]]

View file

@ -0,0 +1,63 @@
with import ./config.nix;
rec {
impure = mkDerivation {
name = "impure";
outputs = [ "out" "stuff" ];
buildCommand =
''
echo impure
x=$(< $TEST_ROOT/counter)
mkdir $out $stuff
echo $x > $out/n
ln -s $out/n $stuff/bla
printf $((x + 1)) > $TEST_ROOT/counter
'';
__impure = true;
impureEnvVars = [ "TEST_ROOT" ];
};
impureOnImpure = mkDerivation {
name = "impure-on-impure";
buildCommand =
''
echo impure-on-impure
x=$(< ${impure}/n)
mkdir $out
printf X$x > $out/n
ln -s ${impure.stuff} $out/symlink
ln -s $out $out/self
'';
__impure = true;
};
# This is not allowed.
inputAddressed = mkDerivation {
name = "input-addressed";
buildCommand =
''
cat ${impure} > $out
'';
};
contentAddressed = mkDerivation {
name = "content-addressed";
buildCommand =
''
echo content-addressed
x=$(< ${impureOnImpure}/n)
printf ''${x:0:1} > $out
'';
outputHashMode = "recursive";
outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ=";
};
inputAddressedAfterCA = mkDerivation {
name = "input-addressed-after-ca";
buildCommand =
''
cat ${contentAddressed} > $out
'';
};
}

View file

@ -0,0 +1,57 @@
source common.sh
requireDaemonNewerThan "2.8pre20220311"
enableFeatures "ca-derivations ca-references impure-derivations"
restartDaemon
set -o pipefail
clearStore
# Basic test of impure derivations: building one a second time should not use the previous result.
printf 0 > $TEST_ROOT/counter
json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
path1=$(echo $json | jq -r .[].outputs.out)
path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
[[ $(< $path1/n) = 0 ]]
[[ $(< $path1_stuff/bla) = 0 ]]
[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
[[ $(< $path2/n) = 1 ]]
# Test impure derivations that depend on impure derivations.
path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path3/n) = X2 ]]
path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path4/n) = X3 ]]
# Test that (self-)references work.
[[ $(< $path4/symlink/bla) = 3 ]]
[[ $(< $path4/self/n) = X3 ]]
# Input-addressed derivations cannot depend on impure derivations directly.
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
# Fixed-output derivations *can* depend on impure derivations.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# And they should not be rebuilt.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations.
path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out)
[[ $(< $path6) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]

View file

@ -98,7 +98,8 @@ nix_tests = \
nix-profile.sh \ nix-profile.sh \
suggestions.sh \ suggestions.sh \
store-ping.sh \ store-ping.sh \
fetchClosure.sh fetchClosure.sh \
impure-derivations.sh
ifeq ($(HAVE_LIBCPUID), 1) ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh nix_tests += compute-levels.sh