scriptis/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge pull request #3795 from obsidiansystems/optional-derivation-output-storepath

Browse source 
Only store hash in DerivationOutput for fixed output derivations
This commit is contained in:
Eelco Dolstra 2020-07-27 16:48:41 +02:00 committed by GitHub
parent d7c0f094cb d5bb67cfa4
commit a5f7d310dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 204 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
perl/lib/Nix/Store.xs
View file

@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath)
HV * outputs = newHV(); HV * outputs = newHV();
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0); hv_store(
outputs, i.first.c_str(), i.first.size(),
newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
0);
hv_stores(hash, "outputs", newRV((SV *) outputs)); hv_stores(hash, "outputs", newRV((SV *) outputs));
AV * inputDrvs = newAV(); AV * inputDrvs = newAV();

2
src/libexpr/get-drvs.cc
View file

@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
outPath = store->printStorePath(i->second.path); outPath = store->printStorePath(i->second.path(*store, drv.name));
} }

29
src/libexpr/primops.cc
View file

@ -52,7 +52,7 @@ void EvalState::realiseContext(const PathSet & context)
DerivationOutputs::iterator i = drv.outputs.find(outputName); DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path)); allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
} }
} }
} }
@ -91,8 +91,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
Path realPath = state.checkSourcePath(state.toRealPath(path, context)); Path realPath = state.checkSourcePath(state.toRealPath(path, context));
// FIXME // FIXME
if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
Derivation drv = readDerivation(*state.store, realPath); if (!state.store->isStorePath(path))
return std::nullopt;
auto storePath = state.store->parseStorePath(path);
if (!(state.store->isValidPath(storePath) && isDerivation(path)))
return std::nullopt;
return storePath;
};
if (auto optStorePath = isValidDerivationInStore()) {
auto storePath = *optStorePath;
Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
Value & w = *state.allocValue(); Value & w = *state.allocValue();
state.mkAttrs(w, 3 + drv.outputs.size()); state.mkAttrs(w, 3 + drv.outputs.size());
Value * v2 = state.allocAttr(w, state.sDrvPath); Value * v2 = state.allocAttr(w, state.sDrvPath);
@ -106,7 +115,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
for (const auto & o : drv.outputs) { for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first)); v2 = state.allocAttr(w, state.symbols.create(o.first));
mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue(); outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first); mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
} }
@ -570,6 +579,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Build the derivation expression by processing the attributes. */ /* Build the derivation expression by processing the attributes. */
Derivation drv; Derivation drv;
drv.name = drvName;
PathSet context; PathSet context;
@ -764,11 +774,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.outputs.insert_or_assign("out", DerivationOutput {
.path = std::move(outPath), .output = DerivationOutputFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = ingestionMethod, .method = ingestionMethod,
.hash = std::move(h), .hash = std::move(h),
}, },
},
}); });
} }
@ -783,8 +794,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = ""; if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput { DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = StorePath::dummy, .path = StorePath::dummy,
.hash = std::optional<FixedOutputHash> {}, },
}); });
} }
@ -795,8 +807,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput { DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath), .path = std::move(outPath),
.hash = std::optional<FixedOutputHash>(), },
}); });
} }
} }
@ -817,7 +830,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)), mkString(*state.allocAttr(v, state.symbols.create(i.first)),
state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
} }
v.attrs->sort(); v.attrs->sort();
} }

76
src/libstore/build.cc
View file

@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
{ {
this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
state = &DerivationGoal::haveDerivation; state = &DerivationGoal::haveDerivation;
name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
trace("created"); trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false; retrySubstitution = false;
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
worker.store.addTempRoot(i.second.path); worker.store.addTempRoot(i.second.path(worker.store, drv->name));
/* Check what outputs paths are not already valid. */ /* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure()
StorePathSet outputClosure; StorePathSet outputClosure;
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue; if (!wantOutput(i.first, wantedOutputs)) continue;
worker.store.computeFSClosure(i.second.path, outputClosure); worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
} }
/* Filter out our own outputs (which we have already checked). */ /* Filter out our own outputs (which we have already checked). */
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
outputClosure.erase(i.second.path); outputClosure.erase(i.second.path(worker.store, drv->name));
/* Get all dependencies of this derivation so that we know which /* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output derivation is responsible for which path in the output
@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure()
if (i.isDerivation()) { if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i); Derivation drv = worker.store.derivationFromPath(i);
for (auto & j : drv.outputs) for (auto & j : drv.outputs)
outputsToDrv.insert_or_assign(j.second.path, i); outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
} }
/* Check each path (slow!). */ /* Check each path (slow!). */
@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised()
for (auto & j : i.second) { for (auto & j : i.second) {
auto k = inDrv.outputs.find(j); auto k = inDrv.outputs.find(j);
if (k != inDrv.outputs.end()) if (k != inDrv.outputs.end())
worker.store.computeFSClosure(k->second.path, inputPaths); worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
else else
throw Error( throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'", "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild()
goal can start a build, and if not, the main loop will sleep a goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */ few seconds and then retry this goal. */
PathSet lockFiles; PathSet lockFiles;
for (auto & outPath : drv->outputPaths()) for (auto & outPath : drv->outputPaths(worker.store))
lockFiles.insert(worker.store.Store::toRealPath(outPath)); lockFiles.insert(worker.store.Store::toRealPath(outPath));
if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!outputLocks.lockPaths(lockFiles, "", false)) {
@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild()
return; return;
} }
missingPaths = drv->outputPaths(); missingPaths = drv->outputPaths(worker.store);
if (buildMode != bmCheck) if (buildMode != bmCheck)
for (auto & i : validPaths) missingPaths.erase(i); for (auto & i : validPaths) missingPaths.erase(i);
/* If any of the outputs already exist but are not valid, delete /* If any of the outputs already exist but are not valid, delete
them. */ them. */
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
if (worker.store.isValidPath(i.second.path)) continue; if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
deletePath(worker.store.Store::toRealPath(i.second.path)); deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
} }
/* Don't do a remote build if the derivation has the attribute /* Don't do a remote build if the derivation has the attribute
@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone()
fmt("running post-build-hook '%s'", settings.postBuildHook), fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)}); Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id); PushActivity pact(act.id);
auto outputPaths = drv->outputPaths(); auto outputPaths = drv->outputPaths(worker.store);
std::map<std::string, std::string> hookEnvironment = getEnv(); std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
if (j.isDerivation()) { if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j); Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputs) for (auto & k : drv.outputs)
worker.store.computeFSClosure(k.second.path, paths); worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
} }
} }
@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder()
/* Substitute output placeholders with the actual output paths. */ /* Substitute output placeholders with the actual output paths. */
for (auto & output : drv->outputs) for (auto & output : drv->outputs)
inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
/* Construct the environment passed to the builder. */ /* Construct the environment passed to the builder. */
initEnv(); initEnv();
@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder()
(typically the dependencies of /bin/sh). Throw them (typically the dependencies of /bin/sh). Throw them
out. */ out. */
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
dirsInChroot.erase(worker.store.printStorePath(i.second.path)); dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
#elif __APPLE__ #elif __APPLE__
/* We don't really have any parent prep work to do (yet?) /* We don't really have any parent prep work to do (yet?)
@ -2613,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */ /* Add an "outputs" object containing the output paths. */
nlohmann::json outputs; nlohmann::json outputs;
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
json["outputs"] = outputs; json["outputs"] = outputs;
/* Handle exportReferencesGraph. */ /* Handle exportReferencesGraph. */
@ -2817,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore
auto drv = derivationFromPath(path.path); auto drv = derivationFromPath(path.path);
for (auto & output : drv.outputs) for (auto & output : drv.outputs)
if (wantOutput(output.first, path.outputs)) if (wantOutput(output.first, path.outputs))
newPaths.insert(output.second.path); newPaths.insert(output.second.path(*this, drv.name));
} else if (!goal.isAllowed(path.path)) } else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
} }
@ -3579,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
if (store.isStorePath(i)) if (store.isStorePath(i))
result.insert(store.parseStorePath(i)); result.insert(store.parseStorePath(i));
else if (drv.outputs.count(i)) else if (drv.outputs.count(i))
result.insert(drv.outputs.find(i)->second.path); result.insert(drv.outputs.find(i)->second.path(store, drv.name));
else throw BuildError("derivation contains an illegal reference specifier '%s'", i); else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
} }
return result; return result;
@ -3617,7 +3617,7 @@ void DerivationGoal::registerOutputs()
if (hook) { if (hook) {
bool allValid = true; bool allValid = true;
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
if (!worker.store.isValidPath(i.second.path)) allValid = false; if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
if (allValid) return; if (allValid) return;
} }
@ -3638,23 +3638,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */ Nix calls. */
StorePathSet referenceablePaths; StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p); for (auto & p : inputPaths) referenceablePaths.insert(p);
for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
for (auto & p : addedPaths) referenceablePaths.insert(p); for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each /* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all output path to determine what other paths it references. Also make all
output paths read-only. */ output paths read-only. */
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
auto path = worker.store.printStorePath(i.second.path); auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
if (!missingPaths.count(i.second.path)) continue; if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
Path actualPath = path; Path actualPath = path;
if (needsHashRewrite()) { if (needsHashRewrite()) {
auto r = redirectedOutputs.find(i.second.path); auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
if (r != redirectedOutputs.end()) { if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second); auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair if (buildMode == bmRepair
&& redirectedBadOutputs.count(i.second.path) && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
&& pathExists(redirected)) && pathExists(redirected))
replaceValidPath(path, redirected); replaceValidPath(path, redirected);
if (buildMode == bmCheck) if (buildMode == bmCheck)
@ -3723,7 +3723,9 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) { if (fixedOutput) {
if (i.second.hash->method == FileIngestionMethod::Flat) { FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
@ -3734,13 +3736,13 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by /* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */ the derivation to its content-addressed location. */
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive Hash h2 = outputHash.method == FileIngestionMethod::Recursive
? hashPath(*i.second.hash->hash.type, actualPath).first ? hashPath(*outputHash.hash.type, actualPath).first
: hashFile(*i.second.hash->hash.type, actualPath); : hashFile(*outputHash.hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
if (i.second.hash->hash != h2) { if (outputHash.hash != h2) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */
@ -3748,7 +3750,7 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true), outputHash.hash.to_string(SRI, true),
h2.to_string(SRI, true))); h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest); Path actualDest = worker.store.Store::toRealPath(dest);
@ -3770,7 +3772,7 @@ void DerivationGoal::registerOutputs()
assert(worker.store.parseStorePath(path) == dest); assert(worker.store.parseStorePath(path) == dest);
ca = FixedOutputHash { ca = FixedOutputHash {
.method = i.second.hash->method, .method = outputHash.method,
.hash = h2, .hash = h2,
}; };
} }
@ -3894,7 +3896,7 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */ /* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
auto path = worker.store.printStorePath(i.second.path); auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
Path prev = path + checkSuffix; Path prev = path + checkSuffix;
deletePath(prev); deletePath(prev);
Path dst = path + checkSuffix; Path dst = path + checkSuffix;
@ -3912,7 +3914,7 @@ void DerivationGoal::registerOutputs()
if the result was not determistic? */ if the result was not determistic? */
if (curRound == nrRounds) { if (curRound == nrRounds) {
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
deletePath(prev); deletePath(prev);
} }
} }
@ -4213,9 +4215,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
for (auto & i : drv->outputs) { for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue; if (!wantOutput(i.first, wantedOutputs)) continue;
bool good = bool good =
worker.store.isValidPath(i.second.path) && worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
(!checkHash || worker.pathContentsGood(i.second.path)); (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
if (good == returnValid) result.insert(i.second.path); if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
} }
return result; return result;
} }

11
src/libstore/builtins/fetchurl.cc
View file

@ -63,16 +63,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto & output = drv.outputs.begin()->second; auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */ /* Try the hashed mirrors first. */
if (output.hash && output.hash->method == FileIngestionMethod::Flat) if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) {
for (auto hashedMirror : settings.hashedMirrors.get()) if (hash->hash.method == FileIngestionMethod::Flat) {
for (auto hashedMirror : settings.hashedMirrors.get()) {
try { try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto & h = output.hash->hash; fetch(hashedMirror + printHashType(*hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return; return;
} catch (Error & e) { } catch (Error & e) {
debug(e.what()); debug(e.what());
} }
}
}
}
/* Otherwise try the specified URL. */ /* Otherwise try the specified URL. */
fetch(mainUrl); fetch(mainUrl);

2
src/libstore/daemon.cc
View file

@ -451,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopBuildDerivation: { case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from)); auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv; BasicDerivation drv;
readDerivation(from, *store, drv); readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from); BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork(); logger->startWork();
if (!trusted) if (!trusted)

103
src/libstore/derivations.cc
View file

@ -7,6 +7,22 @@
namespace nix { namespace nix {
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) {
return doi.path;
},
[&](DerivationOutputFixed dof) {
return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
}
}, output);
}
bool BasicDerivation::isBuiltin() const bool BasicDerivation::isBuiltin() const
{ {
@ -99,7 +115,6 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
expect(str, ","); const auto hash = parseString(str); expect(str, ","); const auto hash = parseString(str);
expect(str, ")"); expect(str, ")");
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") { if (string(hashAlgo, 0, 2) == "r:") {
@ -107,22 +122,29 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
const HashType hashType = parseHashType(hashAlgo); const HashType hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
},
}
};
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path), .path = std::move(path),
.hash = std::move(fsh), }
}; };
} }
static Derivation parseDerivation(const Store & store, std::string && s) static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{ {
Derivation drv; Derivation drv;
drv.name = name;
std::istringstream str(std::move(s)); std::istringstream str(std::move(s));
expect(str, "Derive(["); expect(str, "Derive([");
@ -166,10 +188,10 @@ static Derivation parseDerivation(const Store & store, std::string && s)
} }
Derivation readDerivation(const Store & store, const Path & drvPath) Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{ {
try { try {
return parseDerivation(store, readFile(drvPath)); return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) { } catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
} }
@ -187,7 +209,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
{ {
auto accessor = getFSAccessor(); auto accessor = getFSAccessor();
try { try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) { } catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
} }
@ -255,10 +277,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) { for (auto & i : outputs) {
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first); s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
s += ','; printUnquotedString(s, s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
} else {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
s += ')'; s += ')';
} }
@ -314,7 +340,7 @@ bool BasicDerivation::isFixedOutput() const
{ {
return outputs.size() == 1 && return outputs.size() == 1 &&
outputs.begin()->first == "out" && outputs.begin()->first == "out" &&
outputs.begin()->second.hash; std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output);
} }
@ -346,10 +372,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) { if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin(); DerivationOutputs::const_iterator i = drv.outputs.begin();
auto hash = std::get<DerivationOutputFixed>(i->second.output);
return hashString(htSHA256, "fixed:out:" return hashString(htSHA256, "fixed:out:"
+ i->second.hash->printMethodAlgo() + ":" + hash.hash.printMethodAlgo() + ":"
+ i->second.hash->hash.to_string(Base16, false) + ":" + hash.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path)); + store.printStorePath(i->second.path(store, drv.name)));
} }
/* For other derivations, replace the inputs paths with recursive /* For other derivations, replace the inputs paths with recursive
@ -383,11 +410,11 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
} }
StorePathSet BasicDerivation::outputPaths() const StorePathSet BasicDerivation::outputPaths(const Store & store) const
{ {
StorePathSet paths; StorePathSet paths;
for (auto & i : outputs) for (auto & i : outputs)
paths.insert(i.second.path); paths.insert(i.second.path(store, name));
return paths; return paths;
} }
@ -397,7 +424,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
auto hashAlgo = readString(in); auto hashAlgo = readString(in);
auto hash = readString(in); auto hash = readString(in);
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") { if (string(hashAlgo, 0, 2) == "r:") {
@ -405,15 +431,19 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
auto hashType = parseHashType(hashAlgo); auto hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash { return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash(hash, hashType), .hash = Hash(hash, hashType),
}; },
} }
};
} else
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path), .path = std::move(path),
.hash = std::move(fsh), }
}; };
} }
@ -426,8 +456,19 @@ StringSet BasicDerivation::outputNames() const
} }
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
nameWithSuffix.remove_suffix(extension.size());
return nameWithSuffix;
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
{ {
drv.name = name;
drv.outputs.clear(); drv.outputs.clear();
auto nr = readNum<size_t>(in); auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) { for (size_t n = 0; n < nr; n++) {
@ -456,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
out << i.first out << i.first
<< store.printStorePath(i.second.path); << store.printStorePath(i.second.path(store, drv.name));
if (i.second.hash) { if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
out << i.second.hash->printMethodAlgo() out << hash->hash.printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false); << hash->hash.hash.to_string(Base16, false);
} else { } else {
out << "" << ""; out << "" << "";
} }

23
src/libstore/derivations.hh
View file

@ -13,10 +13,20 @@ namespace nix {
/* Abstract syntax of derivations. */ /* Abstract syntax of derivations. */
struct DerivationOutput struct DerivationOutputInputAddressed
{ {
StorePath path; StorePath path;
std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ };
struct DerivationOutputFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
};
struct DerivationOutput
{
std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
StorePath path(const Store & store, std::string_view drvName) const;
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -35,6 +45,7 @@ struct BasicDerivation
Path builder; Path builder;
Strings args; Strings args;
StringPairs env; StringPairs env;
std::string name;
BasicDerivation() { } BasicDerivation() { }
virtual ~BasicDerivation() { }; virtual ~BasicDerivation() { };
@ -45,10 +56,12 @@ struct BasicDerivation
bool isFixedOutput() const; bool isFixedOutput() const;
/* Return the output paths of a derivation. */ /* Return the output paths of a derivation. */
StorePathSet outputPaths() const; StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */ /* Return the output names of a derivation. */
StringSet outputNames() const; StringSet outputNames() const;
static std::string_view nameFromPath(const StorePath & storePath);
}; };
struct Derivation : BasicDerivation struct Derivation : BasicDerivation
@ -72,7 +85,7 @@ StorePath writeDerivation(ref<Store> store,
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */ /* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath); Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
// FIXME: remove // FIXME: remove
bool isDerivation(const string & fileName); bool isDerivation(const string & fileName);
@ -89,7 +102,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source; struct Source;
struct Sink; struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string & outputName); std::string hashPlaceholder(const std::string & outputName);

11
src/libstore/local-store.cc
View file

@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
DerivationOutputs::const_iterator out = drv.outputs.find("out"); DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end()) if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
} }
else { else {
Hash h = hashDerivationModulo(*this, drv, true); Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
} }
} }
@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtAddDerivationOutput.use() state.stmtAddDerivationOutput.use()
(id) (id)
(i.first) (i.first)
(printStorePath(i.second.path)) (printStorePath(i.second.path(*this, drv.name)))
.exec(); .exec();
} }
} }

4
src/libstore/misc.cc
View file

@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
PathSet invalid; PathSet invalid;
for (auto & j : drv->outputs) for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs) if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.path)) && !isValidPath(j.second.path(*this, drv->name)))
invalid.insert(printStorePath(j.second.path)); invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return; if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {

10
src/nix-store/nix-store.cc
View file

@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j); store2->printStorePath(path.path), j);
auto outPath = store2->printStorePath(i->second.path); auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
if (store2) { if (store2) {
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
auto drv = store->derivationFromPath(storePath); auto drv = store->derivationFromPath(storePath);
StorePathSet outputs; StorePathSet outputs;
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
outputs.insert(i.second.path); outputs.insert(i.second.path(*store, drv.name));
return outputs; return outputs;
} }
else return {storePath}; else return {storePath};
@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
if (forceRealise) realisePath({i2}); if (forceRealise) realisePath({i2});
Derivation drv = store->derivationFromPath(i2); Derivation drv = store->derivationFromPath(i2);
for (auto & j : drv.outputs) for (auto & j : drv.outputs)
cout << fmt("%1%\n", store->printStorePath(j.second.path)); cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
} }
break; break;
} }
@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs)
if (!writeAllowed) throw Error("building paths is not allowed"); if (!writeAllowed) throw Error("building paths is not allowed");
auto drvPath = store->parseStorePath(readString(in)); // informational only auto drvPath = store->parseStorePath(readString(in));
BasicDerivation drv; BasicDerivation drv;
readDerivation(in, *store, drv); readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath));
getBuildSettings(); getBuildSettings();

6
src/nix/develop.cc
View file

@ -130,14 +130,16 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drvName += "-env"; drvName += "-env";
for (auto & output : drv.outputs) for (auto & output : drv.outputs)
drv.env.erase(output.first); drv.env.erase(output.first);
drv.outputs = {{"out", DerivationOutput { .path = StorePath::dummy }}}; drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
drv.env["out"] = ""; drv.env["out"] = "";
drv.env["_outputs_saved"] = drv.env["outputs"]; drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out"; drv.env["outputs"] = "out";
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true); Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName); auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
.path = shellOutPath
} });
drv.env["out"] = store->printStorePath(shellOutPath); drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName); auto shellDrvPath2 = writeDerivation(store, drv, drvName);

5
src/nix/installables.cc
View file

@ -304,8 +304,9 @@ struct InstallableStorePath : Installable
{ {
if (storePath.isDerivation()) { if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
for (auto & [name, output] : store->readDerivation(storePath).outputs) auto drv = store->readDerivation(storePath);
outputs.emplace(name, output.path); for (auto & [name, output] : drv.outputs)
outputs.emplace(name, output.path(*store, drv.name));
return { return {
Buildable { Buildable {
.drvPath = storePath, .drvPath = storePath,

4
src/nix/repl.cc
View file

@ -483,10 +483,10 @@ bool NixRepl::processLine(string line)
but doing it in a child makes it easier to recover from but doing it in a child makes it easier to recover from
problems / SIGINT. */ problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
auto drv = readDerivation(*state->store, drvPath); auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
} }
} else if (command == ":i") { } else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});

8
src/nix/show-derivation.cc
View file

@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand
auto outputsObj(drvObj.object("outputs")); auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first)); auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path)); outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
if (output.second.hash) { if (auto hash = std::get_if<DerivationOutputFixed>(&output.second.output)) {
outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); outputObj.attr("hashAlgo", hash->hash.printMethodAlgo());
outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); outputObj.attr("hash", hash->hash.hash.to_string(Base16, false));
} }
} }
} }