diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 9c55526a3..d0f1b09ca 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,6 +370,33 @@ false.
+ hashed-mirrors
+
+ A list of web servers used by
+ builtins.fetchurl to obtain files by hash.
+ Given a hash type ht and a base-16 hash
+ h, Nix will try to download the file
+ from
+ hashed-mirror/ht/h.
+ This allows files to be downloaded even if they have disappeared
+ from their original URI. For example, given the hashed mirror
+ http://tarballs.example.com/, when building the
+ derivation
+
+
+builtins.fetchurl {
+ url = "https://example.org/foo-1.2.3.tar.xz";
+ sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+}
+
+
+ Nix will attempt to download this file from
+ http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
+ first. If it is not available there, if will try the original URI.
+
+
+
+
http-connections
The maximum number of parallel TCP connections
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index c412e4640..4e038866e 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
+ auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
+ auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
@@ -224,7 +224,7 @@ SV * hashString(char * algo, int base32, char * s)
SV * convertHash(char * algo, char * s, int toBase32)
PPCODE:
try {
- Hash h(s, parseHashType(algo));
+ auto h = Hash::parseAny(s, parseHashType(algo));
string s = h.to_string(toBase32 ? Base32 : Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
@@ -285,7 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE:
try {
- Hash h(hash, parseHashType(algo));
+ auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
@@ -303,10 +303,10 @@ SV * derivationFromPath(char * drvPath)
hash = newHV();
HV * outputs = newHV();
- for (auto & i : drv.outputs)
+ for (auto & i : drv.outputsAndPaths(*store()))
hv_store(
outputs, i.first.c_str(), i.first.size(),
- newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+ newSVpv(store()->printStorePath(i.second.second).c_str(), 0),
0);
hv_stores(hash, "outputs", newRV((SV *) outputs));
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 3579d8fff..ce5127113 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -38,9 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
-static bool allSupportedLocally(const std::set& requiredFeatures) {
+static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) {
for (auto & feature : requiredFeatures)
- if (!settings.systemFeatures.get().count(feature)) return false;
+ if (!store.systemFeatures.get().count(feature)) return false;
return true;
}
@@ -103,10 +103,10 @@ static int _main(int argc, char * * argv)
drvPath = store->parseStorePath(readString(source));
auto requiredFeatures = readStrings>(source);
- auto canBuildLocally = amWilling
+ auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
- && allSupportedLocally(requiredFeatures);
+ && allSupportedLocally(*store, requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
@@ -170,7 +170,45 @@ static int _main(int argc, char * * argv)
if (rightType && !canBuildLocally)
std::cerr << "# postpone\n";
else
+ {
+ // build the hint template.
+ string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)";
+ hintstring += "\n%s available machines:";
+ hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+ for (unsigned int i = 0; i < machines.size(); ++i) {
+ hintstring += "\n(%s, %s, %s, %s)";
+ }
+
+ // add the template values.
+ string drvstr;
+ if (drvPath.has_value())
+ drvstr = drvPath->to_string();
+ else
+ drvstr = "";
+
+ auto hint = hintformat(hintstring);
+ hint
+ % drvstr
+ % neededSystem
+ % concatStringsSep(", ", requiredFeatures)
+ % machines.size();
+
+ for (auto & m : machines) {
+ hint % concatStringsSep>(", ", m.systemTypes)
+ % m.maxJobs
+ % concatStringsSep(", ", m.supportedFeatures)
+ % concatStringsSep(", ", m.mandatoryFeatures);
+ }
+
+ logErrorInfo(lvlInfo, {
+ .name = "Remote build",
+ .description = "Failed to find a machine for remote build!",
+ .hint = hint
+ });
+
std::cerr << "# decline\n";
+ }
break;
}
@@ -186,15 +224,7 @@ static int _main(int argc, char * * argv)
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
- Store::Params storeParams;
- if (hasPrefix(bestMachine->storeUri, "ssh://")) {
- storeParams["max-connections"] = "1";
- storeParams["log-fd"] = "4";
- if (bestMachine->sshKey != "")
- storeParams["ssh-key"] = bestMachine->sshKey;
- }
-
- sshStore = openStore(bestMachine->storeUri, storeParams);
+ sshStore = bestMachine->openStore();
sshStore->connect();
storeUri = bestMachine->storeUri;
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index deb32484f..46177a0a4 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -405,7 +405,7 @@ Value & AttrCursor::forceValue()
return v;
}
-std::shared_ptr AttrCursor::maybeGetAttr(Symbol name)
+std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
{
if (root->db) {
if (!cachedValue)
@@ -422,9 +422,12 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name)
if (attr) {
if (std::get_if(&attr->second))
return nullptr;
- else if (std::get_if(&attr->second))
- throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
- else
+ else if (std::get_if(&attr->second)) {
+ if (forceErrors)
+ debug("reevaluating failed cached attribute '%s'");
+ else
+ throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+ } else
return std::make_shared(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@@ -469,9 +472,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
-std::shared_ptr AttrCursor::getAttr(Symbol name)
+std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors)
{
- auto p = maybeGetAttr(name);
+ auto p = maybeGetAttr(name, forceErrors);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
@@ -600,7 +603,7 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
- auto aDrvPath = getAttr(root->state.sDrvPath);
+ auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index afee85fa9..8ffffc0ed 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -9,6 +9,8 @@
namespace nix::eval_cache {
+MakeError(CachedEvalError, EvalError);
+
class AttrDb;
class AttrCursor;
@@ -92,11 +94,11 @@ public:
std::string getAttrPathStr(Symbol name) const;
- std::shared_ptr maybeGetAttr(Symbol name);
+ std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr maybeGetAttr(std::string_view name);
- std::shared_ptr getAttr(Symbol name);
+ std::shared_ptr getAttr(Symbol name, bool forceErrors = false);
std::shared_ptr getAttr(std::string_view name);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7a2f55504..0123070d1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -345,6 +345,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store)
, sStructuredAttrs(symbols.create("__structuredAttrs"))
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
+ , sContentAddressed(symbols.create("__contentAddressed"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
@@ -1256,10 +1257,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
try {
lambda.body->eval(*this, env2, v);
} catch (Error & e) {
- addErrorTrace(e, lambda.pos, "while evaluating %s",
- (lambda.name.set()
- ? "'" + (string) lambda.name + "'"
- : "anonymous lambdaction"));
+ addErrorTrace(e, lambda.pos, "while evaluating %s",
+ (lambda.name.set()
+ ? "'" + (string) lambda.name + "'"
+ : "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", "");
throw;
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 8986952e3..5855b4ef2 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -74,6 +74,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+ sContentAddressed,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon;
@@ -374,6 +375,9 @@ struct EvalSettings : Config
Setting traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+
+ Setting useEvalCache{this, true, "eval-cache",
+ "Whether to use the flake evaluation cache."};
};
extern EvalSettings evalSettings;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 437b2e749..14bb5eaa9 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -113,9 +113,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
state.mkList(*outputsVal, drv.outputs.size());
unsigned int outputs_index = 0;
- for (const auto & o : drv.outputs) {
+ for (const auto & o : drv.outputsAndPaths(*state.store)) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
- mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
+ mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@@ -583,6 +583,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
+ bool contentAddressed = false;
std::optional outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
@@ -639,9 +640,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (i->value->type == tNull) continue;
}
+ if (i->name == state.sContentAddressed) {
+ settings.requireExperimentalFeature("ca-derivations");
+ contentAddressed = state.forceBool(*i->value, pos);
+ }
+
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
- if (i->name == state.sArgs) {
+ else if (i->name == state.sArgs) {
state.forceList(*i->value, pos);
for (unsigned int n = 0; n < i->value->listSize(); ++n) {
string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@@ -694,7 +700,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
} catch (Error & e) {
- e.addTrace(posDrvName,
+ e.addTrace(posDrvName,
"while evaluating the attribute '%1%' of the derivation '%2%'",
key, drvName);
throw;
@@ -761,7 +767,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
if (outputHash) {
- /* Handle fixed-output derivations. */
+ /* Handle fixed-output derivations.
+
+ Ignore `__contentAddressed` because fixed output derivations are
+ already content addressed. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
@@ -772,9 +781,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
- if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
+ drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
- .output = DerivationOutputFixed {
+ .output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
@@ -783,6 +792,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
+ else if (contentAddressed) {
+ HashType ht = parseHashType(outputHashAlgo);
+ for (auto & i : outputs) {
+ drv.env[i] = hashPlaceholder(i);
+ drv.outputs.insert_or_assign(i, DerivationOutput {
+ .output = DerivationOutputCAFloating {
+ .method = ingestionMethod,
+ .hashType = std::move(ht),
+ },
+ });
+ }
+ }
+
else {
/* Compute a hash over the "masked" store derivation, which is
the final one except that in the list of outputs, the
@@ -791,7 +813,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
that changes in the set of output names do get reflected in
the hash. */
for (auto & i : outputs) {
- if (!jsonObject) drv.env[i] = "";
+ drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputInputAddressed {
@@ -800,11 +822,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
- Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
+ // Regular, non-CA derivation should always return a single hash and not
+ // hash per output.
+ Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
- if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
+ drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputInputAddressed {
@@ -815,7 +839,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
/* Write the resulting term into the Nix store directory. */
- auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+ auto drvPath = writeDerivation(state.store, drv, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);
printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -828,9 +852,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
- for (auto & i : drv.outputs) {
+ for (auto & i : drv.outputsAndPaths(*state.store)) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
- state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
+ state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort();
}
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index fc2a6a1c2..cef85cfef 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
else
ref = value;
}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index cddcf0e59..0dbf4ae1d 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
+ *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 28db8aa9c..eaa635595 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -130,12 +130,12 @@ std::pair Input::fetch(ref store) const
tree.actualPath = store->toRealPath(tree.storePath);
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
- input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
+ input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
}
if (auto prevLastModified = getLastModified()) {
@@ -200,9 +200,12 @@ std::string Input::getType() const
std::optional Input::getNarHash() const
{
- if (auto s = maybeGetStrAttr(attrs, "narHash"))
- // FIXME: require SRI hash.
- return newHashAllowEmpty(*s, htSHA256);
+ if (auto s = maybeGetStrAttr(attrs, "narHash")) {
+ auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
+ if (hash.type != htSHA256)
+ throw UsageError("narHash must use SHA-256");
+ return hash;
+ }
return {};
}
@@ -216,7 +219,7 @@ std::optional Input::getRef() const
std::optional Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
- return Hash(*s, htSHA1);
+ return Hash::parseAny(*s, htSHA1);
return {};
}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 5d38e0c2b..5ca0f8521 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -121,7 +121,7 @@ struct GitInputScheme : InputScheme
args.push_back(*ref);
}
- if (input.getRev()) throw Error("cloning a specific revision is not implemented");
+ if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented");
args.push_back(destDir);
@@ -269,7 +269,7 @@ struct GitInputScheme : InputScheme
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
- haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
+ haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
@@ -293,14 +293,14 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
input.attrs.insert_or_assign("rev",
- Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
+ Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme
}
if (!input.getRev())
- input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -421,7 +421,7 @@ struct GitInputScheme : InputScheme
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
- auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
+ auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8bb7c2c1d..9f84ffb68 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -29,7 +29,7 @@ struct GitArchiveInputScheme : InputScheme
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
@@ -41,7 +41,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
@@ -191,7 +191,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string { json["sha"] }, htSHA1);
+ auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
@@ -235,7 +235,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+ auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 91dc83740..b981d4d8e 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -18,7 +18,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
- rev = Hash(path[1], htSHA1);
+ rev = Hash::parseAny(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
@@ -29,7 +29,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index c48cb6fd1..3e76ffc4d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -209,7 +209,7 @@ struct MercurialInputScheme : InputScheme
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
- input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 55158cece..a2d16365e 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -67,8 +67,10 @@ DownloadFileResult downloadFile(
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
- info.narHash = hashString(htSHA256, *sink.s);
+ ValidPathInfo info {
+ store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
+ hashString(htSHA256, *sink.s),
+ };
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 3f7d99a1d..be3c06a38 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -362,7 +362,7 @@ public:
auto width = getWindowSize().second;
if (width <= 0) width = std::numeric_limits::max();
- writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
+ writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
}
std::string getStatus(State & state)
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 30150eeba..5433fe50d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -143,7 +143,7 @@ struct FileSource : FdSource
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
- assert(info.narHash && info.narSize);
+ assert(info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
@@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto [fdTemp, fnTemp] = createTempFile();
+ AutoDelete autoDelete(fnTemp);
+
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
@@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
+ fileSink.flush();
}
auto now2 = std::chrono::steady_clock::now();
@@ -216,7 +219,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}
}
- upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
+ upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
}
/* Optionally maintain an index of DWARF debug info files
@@ -280,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url,
- std::make_shared(fnTemp, std::ios_base::in),
+ std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
@@ -309,14 +312,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast();
- uint64_t narSize = 0;
+ LengthSink narSize;
+ TeeSink tee { sink, narSize };
- LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
- sink(data, len);
- narSize += len;
- });
-
- auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
+ auto decompressor = makeDecompressionSink(info->compression, tee);
try {
getFile(info->url, *decompressor);
@@ -328,7 +327,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
stats.narRead++;
//stats.narReadCompressedBytes += nar->size(); // FIXME
- stats.narReadBytes += narSize;
+ stats.narReadBytes += narSize.length;
}
void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
@@ -382,7 +381,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
- ValidPathInfo info(makeFixedOutputPath(method, *h, name));
+ ValidPathInfo info {
+ makeFixedOutputPath(method, *h, name),
+ Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ };
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
@@ -393,7 +395,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
- ValidPathInfo info(computeStorePathForText(name, s, references));
+ ValidPathInfo info {
+ computeStorePathForText(name, s, references),
+ Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ };
info.references = references;
if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4156cd678..afb2bb096 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -806,8 +806,8 @@ private:
/* RAII object to delete the chroot directory. */
std::shared_ptr autoDelChroot;
- /* Whether this is a fixed-output derivation. */
- bool fixedOutput;
+ /* The sort of derivation we are building. */
+ DerivationType derivationType;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
@@ -1181,8 +1181,8 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false;
- for (auto & i : drv->outputs)
- worker.store.addTempRoot(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ worker.store.addTempRoot(i.second.second);
/* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@@ -1195,9 +1195,9 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique(drvPath, *drv);
- if (parsedDrv->contentAddressed()) {
+ if (drv->type() == DerivationType::CAFloating) {
settings.requireExperimentalFeature("ca-derivations");
- throw Error("ca-derivations isn't implemented yet");
+ throw UnimplementedError("ca-derivations isn't implemented yet");
}
@@ -1288,14 +1288,14 @@ void DerivationGoal::repairClosure()
/* Get the output closure. */
StorePathSet outputClosure;
- for (auto & i : drv->outputs) {
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
if (!wantOutput(i.first, wantedOutputs)) continue;
- worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
+ worker.store.computeFSClosure(i.second.second, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
- for (auto & i : drv->outputs)
- outputClosure.erase(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ outputClosure.erase(i.second.second);
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
@@ -1306,8 +1306,8 @@ void DerivationGoal::repairClosure()
for (auto & i : inputClosure)
if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i);
- for (auto & j : drv.outputs)
- outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
+ for (auto & j : drv.outputsAndPaths(worker.store))
+ outputsToDrv.insert_or_assign(j.second.second, i);
}
/* Check each path (slow!). */
@@ -1392,12 +1392,12 @@ void DerivationGoal::inputsRealised()
debug("added input paths %s", worker.store.showPaths(inputPaths));
- /* Is this a fixed-output derivation? */
- fixedOutput = drv->isFixedOutput();
+ /* What type of derivation are we building? */
+ derivationType = drv->type();
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
- nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1;
+ nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a
@@ -1466,16 +1466,16 @@ void DerivationGoal::tryToBuild()
/* If any of the outputs already exist but are not valid, delete
them. */
- for (auto & i : drv->outputs) {
- if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
- debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
- deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ if (worker.store.isValidPath(i.second.second)) continue;
+ debug("removing invalid path '%s'", worker.store.printStorePath(i.second.second));
+ deletePath(worker.store.Store::toRealPath(i.second.second));
}
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
- bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
+ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
/* Is the build hook willing to accept this job? */
if (!buildLocally) {
@@ -1783,7 +1783,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
- fixedOutput || diskFull ? BuildResult::TransientFailure :
+ derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@@ -1919,8 +1919,8 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
for (auto & j : paths2) {
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
- for (auto & k : drv.outputs)
- worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
+ for (auto & k : drv.outputsAndPaths(worker.store))
+ worker.store.computeFSClosure(k.second.second, paths);
}
}
@@ -1964,13 +1964,13 @@ void linkOrCopy(const Path & from, const Path & to)
void DerivationGoal::startBuilder()
{
/* Right platform? */
- if (!parsedDrv->canBuildLocally())
+ if (!parsedDrv->canBuildLocally(worker.store))
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
drv->platform,
concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
worker.store.printStorePath(drvPath),
settings.thisSystem,
- concatStringsSep(", ", settings.systemFeatures));
+ concatStringsSep(", ", worker.store.systemFeatures));
if (drv->isBuiltin())
preloadNSS();
@@ -1996,7 +1996,7 @@ void DerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
- useChroot = !fixedOutput && !noChroot;
+ useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
}
if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -2014,8 +2014,8 @@ void DerivationGoal::startBuilder()
chownToBuilder(tmpDir);
/* Substitute output placeholders with the actual output paths. */
- for (auto & output : drv->outputs)
- inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
+ for (auto & output : drv->outputsAndPaths(worker.store))
+ inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.second);
/* Construct the environment passed to the builder. */
initEnv();
@@ -2165,7 +2165,7 @@ void DerivationGoal::startBuilder()
"nogroup:x:65534:\n") % sandboxGid).str());
/* Create /etc/hosts with localhost entry. */
- if (!fixedOutput)
+ if (!(derivationIsImpure(derivationType)))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@@ -2199,8 +2199,8 @@ void DerivationGoal::startBuilder()
rebuilding a path that is in settings.dirsInChroot
(typically the dependencies of /bin/sh). Throw them
out. */
- for (auto & i : drv->outputs)
- dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ dirsInChroot.erase(worker.store.printStorePath(i.second.second));
#elif __APPLE__
/* We don't really have any parent prep work to do (yet?)
@@ -2373,7 +2373,7 @@ void DerivationGoal::startBuilder()
us.
*/
- if (!fixedOutput)
+ if (!(derivationIsImpure(derivationType)))
privateNetwork = true;
userNamespaceSync.create();
@@ -2574,7 +2574,7 @@ void DerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */
- if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
+ if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
@@ -2585,7 +2585,7 @@ void DerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
- if (fixedOutput) {
+ if (derivationIsImpure(derivationType)) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@@ -2612,8 +2612,8 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
- for (auto & i : drv->outputs)
- outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.second), inputRewrites);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
@@ -2815,9 +2815,9 @@ struct RestrictedStore : public LocalFSStore
if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
auto drv = derivationFromPath(path.path);
- for (auto & output : drv.outputs)
+ for (auto & output : drv.outputsAndPaths(*this))
if (wantOutput(output.first, path.outputs))
- newPaths.insert(output.second.path(*this, drv.name));
+ newPaths.insert(output.second.second);
} else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
}
@@ -2920,7 +2920,8 @@ void DerivationGoal::startDaemon()
FdSink to(remote.get());
try {
daemon::processConnection(store, from, to,
- daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
+ daemon::NotTrusted, daemon::Recursive,
+ [&](Store & store) { store.createUser("nobody", 65535); });
debug("terminated daemon connection");
} catch (SysError &) {
ignoreException();
@@ -3179,7 +3180,7 @@ void DerivationGoal::runChild()
createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts");
ss.push_back("/dev/full");
- if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+ if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
ss.push_back("/dev/kvm");
ss.push_back("/dev/null");
ss.push_back("/dev/random");
@@ -3195,7 +3196,7 @@ void DerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
- if (fixedOutput) {
+ if (derivationIsImpure(derivationType)) {
ss.push_back("/etc/resolv.conf");
// Only use nss functions to resolve hosts and
@@ -3436,7 +3437,7 @@ void DerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
- if (fixedOutput)
+ if (derivationIsImpure(derivationType))
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Our rwx outputs */
@@ -3616,8 +3617,8 @@ void DerivationGoal::registerOutputs()
to do anything here. */
if (hook) {
bool allValid = true;
- for (auto & i : drv->outputs)
- if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ if (!worker.store.isValidPath(i.second.second)) allValid = false;
if (allValid) return;
}
@@ -3638,23 +3639,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */
StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p);
- for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store)) referenceablePaths.insert(i.second.second);
for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
- for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
- if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ auto path = worker.store.printStorePath(i.second.second);
+ if (!missingPaths.count(i.second.second)) continue;
Path actualPath = path;
if (needsHashRewrite()) {
- auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
+ auto r = redirectedOutputs.find(i.second.second);
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
- && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
+ && redirectedBadOutputs.count(i.second.second)
&& pathExists(redirected))
replaceValidPath(path, redirected);
if (buildMode == bmCheck)
@@ -3721,9 +3722,22 @@ void DerivationGoal::registerOutputs()
hash). */
std::optional ca;
- if (fixedOutput) {
-
- FixedOutputHash outputHash = std::get(i.second.output).hash;
+ if (! std::holds_alternative(i.second.first.output)) {
+ DerivationOutputCAFloating outputHash;
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ assert(false); // Enclosing `if` handles this case in other branch
+ },
+ [&](DerivationOutputCAFixed dof) {
+ outputHash = DerivationOutputCAFloating {
+ .method = dof.hash.method,
+ .hashType = dof.hash.hash.type,
+ };
+ },
+ [&](DerivationOutputCAFloating dof) {
+ outputHash = dof;
+ },
+ }, i.second.first.output);
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
@@ -3737,12 +3751,17 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHash.method == FileIngestionMethod::Recursive
- ? hashPath(outputHash.hash.type, actualPath).first
- : hashFile(outputHash.hash.type, actualPath);
+ ? hashPath(outputHash.hashType, actualPath).first
+ : hashFile(outputHash.hashType, actualPath);
- auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
+ auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.second.name());
- if (outputHash.hash != h2) {
+ // true if either floating CA, or incorrect fixed hash.
+ bool needsMove = true;
+
+ if (auto p = std::get_if(& i.second.first.output)) {
+ Hash & h = p->hash.hash;
+ if (h != h2) {
/* Throw an error after registering the path as
valid. */
@@ -3750,9 +3769,15 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
- outputHash.hash.to_string(SRI, true),
+ h.to_string(SRI, true),
h2.to_string(SRI, true)));
+ } else {
+ // matched the fixed hash, so no move needed.
+ needsMove = false;
+ }
+ }
+ if (needsMove) {
Path actualDest = worker.store.Store::toRealPath(dest);
if (worker.store.isValidPath(dest))
@@ -3840,8 +3865,10 @@ void DerivationGoal::registerOutputs()
worker.markContentsGood(worker.store.parseStorePath(path));
}
- ValidPathInfo info(worker.store.parseStorePath(path));
- info.narHash = hash.first;
+ ValidPathInfo info {
+ worker.store.parseStorePath(path),
+ hash.first,
+ };
info.narSize = hash.second;
info.references = std::move(references);
info.deriver = drvPath;
@@ -3897,8 +3924,8 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
- for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ auto path = worker.store.printStorePath(i.second.second);
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
@@ -3915,8 +3942,8 @@ void DerivationGoal::registerOutputs()
/* Remove the .check directories if we're done. FIXME: keep them
if the result was not determistic? */
if (curRound == nrRounds) {
- for (auto & i : drv->outputs) {
- Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ Path prev = worker.store.printStorePath(i.second.second) + checkSuffix;
deletePath(prev);
}
}
@@ -4214,12 +4241,12 @@ void DerivationGoal::flushLine()
StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
{
StorePathSet result;
- for (auto & i : drv->outputs) {
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
- worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
- (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
- if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
+ worker.store.isValidPath(i.second.second) &&
+ (!checkHash || worker.pathContentsGood(i.second.second));
+ if (good == returnValid) result.insert(i.second.second);
}
return result;
}
@@ -4852,8 +4879,17 @@ void Worker::run(const Goals & _topGoals)
waitForInput();
else {
if (awake.empty() && 0 == settings.maxBuildJobs)
- throw Error("unable to start any build; either increase '--max-jobs' "
- "or enable remote builds");
+ {
+ if (getMachines().empty())
+ throw Error("unable to start any build; either increase '--max-jobs' "
+ "or enable remote builds."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+ else
+ throw Error("unable to start any build; remote machines may not have "
+ "all required system features."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+
+ }
assert(!awake.empty());
}
}
@@ -5037,7 +5073,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
- HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
+ HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 6585a480d..4fb5d8a06 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,6 +58,20 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
+ /* Try the hashed mirrors first. */
+ if (getAttr("outputHashMode") == "flat")
+ for (auto hashedMirror : settings.hashedMirrors.get())
+ try {
+ if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
+ std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+ Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
+ fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+ return;
+ } catch (Error & e) {
+ debug(e.what());
+ }
+
+ /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index f83b98a98..0885c3d0e 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -1,4 +1,6 @@
+#include "args.hh"
#include "content-address.hh"
+#include "split.hh"
namespace nix {
@@ -24,10 +26,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+ hash.to_string(Base32, true);
}
-// FIXME Put this somewhere?
-template struct overloaded : Ts... { using Ts::operator()...; };
-template overloaded(Ts...) -> overloaded;
-
std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded {
[](TextHash th) {
@@ -40,38 +38,46 @@ std::string renderContentAddress(ContentAddress ca) {
}
ContentAddress parseContentAddress(std::string_view rawCa) {
- auto prefixSeparator = rawCa.find(':');
- if (prefixSeparator != string::npos) {
- auto prefix = string(rawCa, 0, prefixSeparator);
- if (prefix == "text") {
- auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- Hash hash = Hash(string(hashTypeAndHash));
- if (hash.type != htSHA256) {
- throw Error("parseContentAddress: the text hash should have type SHA256");
- }
- return TextHash { hash };
- } else if (prefix == "fixed") {
- // This has to be an inverse of makeFixedOutputCA
- auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- if (methodAndHash.substr(0,2) == "r:") {
- std::string_view hashRaw = methodAndHash.substr(2,string::npos);
- return FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = Hash(string(hashRaw)),
- };
- } else {
- std::string_view hashRaw = methodAndHash;
- return FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = Hash(string(hashRaw)),
- };
- }
- } else {
- throw Error("parseContentAddress: format not recognized; has to be text or fixed");
- }
- } else {
- throw Error("Not a content address because it lacks an appropriate prefix");
+ auto rest = rawCa;
+
+ std::string_view prefix;
+ {
+ auto optPrefix = splitPrefixTo(rest, ':');
+ if (!optPrefix)
+ throw UsageError("not a content address because it is not in the form ':': %s", rawCa);
+ prefix = *optPrefix;
}
+
+ auto parseHashType_ = [&](){
+ auto hashTypeRaw = splitPrefixTo(rest, ':');
+ if (!hashTypeRaw)
+ throw UsageError("content address hash must be in form ':', but found: %s", rawCa);
+ HashType hashType = parseHashType(*hashTypeRaw);
+ return std::move(hashType);
+ };
+
+ // Switch on prefix
+ if (prefix == "text") {
+ // No parsing of the method, "text" only support flat.
+ HashType hashType = parseHashType_();
+ if (hashType != htSHA256)
+ throw Error("text content address hash should use %s, but instead uses %s",
+ printHashType(htSHA256), printHashType(hashType));
+ return TextHash {
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else if (prefix == "fixed") {
+ // Parse method
+ auto method = FileIngestionMethod::Flat;
+ if (splitPrefix(rest, "r:"))
+ method = FileIngestionMethod::Recursive;
+ HashType hashType = parseHashType_();
+ return FixedOutputHash {
+ .method = method,
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else
+ throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
};
std::optional parseContentAddressOpt(std::string_view rawCaOpt) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index b9a750425..ad3fe1847 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -289,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
- to << hash->to_string(Base16, false);
+ to << hash.to_string(Base16, false);
break;
}
@@ -454,8 +454,46 @@ static void performOp(TunnelLogger * logger, ref store,
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
- if (!trusted)
- throw Error("you are not privileged to build derivations");
+
+ /* Content-addressed derivations are trustless because their output paths
+ are verified by their content alone, so any derivation is free to
+ try to produce such a path.
+
+ Input-addressed derivation output paths, however, are calculated
+ from the derivation closure that produced them---even knowing the
+ root derivation is not enough. That the output data actually came
+ from those derivations is fundamentally unverifiable, but the daemon
+ trusts itself on that matter. The question instead is whether the
+ submitted plan has rights to the output paths it wants to fill, and
+ at least the derivation closure proves that.
+
+ It would have been nice if input-address algorithm merely depended
+ on the build time closure, rather than depending on the derivation
+ closure. That would mean input-addressed paths used at build time
+ would just be trusted and not need their own evidence. This is in
+ fact fine as the same guarantees would hold *inductively*: either
+ the remote builder has those paths and already trusts them, or it
+ needs to build them too and thus their evidence must be provided in
+ turn. The advantage of this variant algorithm is that the evidence
+ for input-addressed paths which the remote builder already has
+ doesn't need to be sent again.
+
+ That said, now that we have floating CA derivations, it is better
+ that people just migrate to those which also solve this problem, and
+ others. It's the same migration difficulty with strictly more
+ benefit.
+
+ Lastly, do note that when we parse fixed-output content-addressed
+ derivations, we throw out the precomputed output paths and just
+ store the hashes, so there aren't two competing sources of truth an
+ attacker could exploit. */
+ if (drv.type() == DerivationType::InputAddressed && !trusted)
+ throw Error("you are not privileged to build input-addressed derivations");
+
+ /* Make sure that the non-input-addressed derivations that got this far
+ are in fact content-addressed if we don't trust them. */
+ assert(derivationIsCA(drv.type()) || trusted);
+
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
@@ -638,7 +676,7 @@ static void performOp(TunnelLogger * logger, ref store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
- << info->narHash->to_string(Base16, false);
+ << info->narHash.to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
@@ -688,17 +726,18 @@ static void performOp(TunnelLogger * logger, ref store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
logger->stopWork();
- dumpPath(store->printStorePath(path), to);
+ dumpPath(store->toRealPath(path), to);
break;
}
case wopAddToStoreNar: {
bool repair, dontCheckSigs;
- ValidPathInfo info(store->parseStorePath(readString(from)));
+ auto path = store->parseStorePath(readString(from));
auto deriver = readString(from);
+ auto narHash = Hash::parseAny(readString(from), htSHA256);
+ ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
- info.narHash = Hash(readString(from), htSHA256);
info.references = readStorePaths(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings(from);
@@ -817,8 +856,7 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
- const std::string & userName,
- uid_t userId)
+ std::function authHook)
{
auto monitor = !recursive ? std::make_unique(from.fd) : nullptr;
@@ -859,15 +897,7 @@ void processConnection(
/* If we can't accept clientVersion, then throw an error
*here* (not above). */
-
-#if 0
- /* Prevent users from doing something very dangerous. */
- if (geteuid() == 0 &&
- querySetting("build-users-group", "") == "")
- throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-
- store->createUser(userName, userId);
+ authHook(*store);
tunnelLogger->stopWork();
to.flush();
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index 266932013..841ace316 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -12,7 +12,10 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
- const std::string & userName,
- uid_t userId);
+ /* Arbitrary hook to check authorization / initialize user data / whatever
+ after the protocol has been negotiated. The idea is that this function
+ and everything it calls doesn't know about this stuff, and the
+ `nix-daemon` handles that instead. */
+ std::function authHook);
}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 8815c4632..abbef9aa2 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -7,23 +7,54 @@
namespace nix {
-// FIXME Put this somewhere?
-template struct overloaded : Ts... { using Ts::operator()...; };
-template overloaded(Ts...) -> overloaded;
-
-StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
+std::optional DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
{
return std::visit(overloaded {
- [](DerivationOutputInputAddressed doi) {
- return doi.path;
+ [](DerivationOutputInputAddressed doi) -> std::optional {
+ return { doi.path };
+ },
+ [&](DerivationOutputCAFixed dof) -> std::optional {
+ return {
+ store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
+ };
+ },
+ [](DerivationOutputCAFloating dof) -> std::optional {
+ return std::nullopt;
},
- [&](DerivationOutputFixed dof) {
- return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
- }
}, output);
}
+bool derivationIsCA(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return true;
+ };
+ // Since enums can have non-variant values, but making a `default:` would
+ // disable exhaustiveness warnings.
+ assert(false);
+}
+
+bool derivationIsFixed(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return false;
+ };
+ assert(false);
+}
+
+bool derivationIsImpure(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return false;
+ };
+ assert(false);
+}
+
+
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:";
@@ -31,7 +62,7 @@ bool BasicDerivation::isBuiltin() const
StorePath writeDerivation(ref store,
- const Derivation & drv, std::string_view name, RepairFlag repair)
+ const Derivation & drv, RepairFlag repair)
{
auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs)
@@ -39,7 +70,7 @@ StorePath writeDerivation(ref store,
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
- auto suffix = std::string(name) + drvExtension;
+ auto suffix = std::string(drv.name) + drvExtension;
auto contents = drv.unparse(*store, false);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
@@ -108,29 +139,33 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(const Store & store,
+ StorePath path, std::string_view hashAlgo, std::string_view hash)
{
- expect(str, ","); auto path = store.parseStorePath(parsePath(str));
- expect(str, ","); auto hashAlgo = parseString(str);
- expect(str, ","); const auto hash = parseString(str);
- expect(str, ")");
-
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
- hashAlgo = string(hashAlgo, 2);
+ hashAlgo = hashAlgo.substr(2);
}
const HashType hashType = parseHashType(hashAlgo);
- return DerivationOutput {
- .output = DerivationOutputFixed {
- .hash = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- },
- }
- };
+ return hash != ""
+ ? DerivationOutput {
+ .output = DerivationOutputCAFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+ },
+ }
+ }
+ : (settings.requireExperimentalFeature("ca-derivations"),
+ DerivationOutput {
+ .output = DerivationOutputCAFloating {
+ .method = std::move(method),
+ .hashType = std::move(hashType),
+ },
+ });
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@@ -139,6 +174,16 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
};
}
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+{
+ expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+ expect(str, ","); const auto hashAlgo = parseString(str);
+ expect(str, ","); const auto hash = parseString(str);
+ expect(str, ")");
+
+ return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
+}
+
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
@@ -250,13 +295,20 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
- if (auto hash = std::get_if(&i.second.output)) {
- s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
- s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
- } else {
- s += ','; printUnquotedString(s, "");
- s += ','; printUnquotedString(s, "");
- }
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ },
+ [&](DerivationOutputCAFixed dof) {
+ s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
+ s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+ },
+ [&](DerivationOutputCAFloating dof) {
+ s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+ s += ','; printUnquotedString(s, "");
+ },
+ }, i.second.output);
s += ')';
}
@@ -308,60 +360,134 @@ bool isDerivation(const string & fileName)
}
-bool BasicDerivation::isFixedOutput() const
+DerivationType BasicDerivation::type() const
{
- return outputs.size() == 1 &&
- outputs.begin()->first == "out" &&
- std::holds_alternative(outputs.begin()->second.output);
+ std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
+ std::optional floatingHashType;
+ for (auto & i : outputs) {
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed _) {
+ inputAddressedOutputs.insert(i.first);
+ },
+ [&](DerivationOutputCAFixed _) {
+ fixedCAOutputs.insert(i.first);
+ },
+ [&](DerivationOutputCAFloating dof) {
+ floatingCAOutputs.insert(i.first);
+ if (!floatingHashType) {
+ floatingHashType = dof.hashType;
+ } else {
+ if (*floatingHashType != dof.hashType)
+ throw Error("All floating outputs must use the same hash type");
+ }
+ },
+ }, i.second.output);
+ }
+
+ if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ throw Error("Must have at least one output");
+ } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ return DerivationType::InputAddressed;
+ } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ if (fixedCAOutputs.size() > 1)
+ // FIXME: Experimental feature?
+ throw Error("Only one fixed output is allowed for now");
+ if (*fixedCAOutputs.begin() != "out")
+ throw Error("Single fixed output must be named \"out\"");
+ return DerivationType::CAFixed;
+ } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+ return DerivationType::CAFloating;
+ } else {
+ throw Error("Can't mix derivation output types");
+ }
}
DrvHashes drvHashes;
+/* pathDerivationModulo and hashDerivationModulo are mutually recursive
+ */
-/* Returns the hash of a derivation modulo fixed-output
- subderivations. A fixed-output derivation is a derivation with one
- output (`out') for which an expected hash and hash algorithm are
- specified (using the `outputHash' and `outputHashAlgo'
- attributes). We don't want changes to such derivations to
- propagate upwards through the dependency graph, changing output
- paths everywhere.
+/* Look up the derivation by value and memoize the
+ `hashDerivationModulo` call.
+ */
+static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+{
+ auto h = drvHashes.find(drvPath);
+ if (h == drvHashes.end()) {
+ assert(store.isValidPath(drvPath));
+ // Cache it
+ h = drvHashes.insert_or_assign(
+ drvPath,
+ hashDerivationModulo(
+ store,
+ store.readDerivation(drvPath),
+ false)).first;
+ }
+ return h->second;
+}
- For instance, if we change the url in a call to the `fetchurl'
- function, we do not want to rebuild everything depending on it
- (after all, (the hash of) the file being downloaded is unchanged).
- So the *output paths* should not change. On the other hand, the
- *derivation paths* should change to reflect the new dependency
- graph.
+/* See the header for interface details. These are the implementation details.
- That's what this function does: it returns a hash which is just the
- hash of the derivation ATerm, except that any input derivation
- paths have been replaced by the result of a recursive call to this
- function, and that for fixed-output derivations we return a hash of
- its output path. */
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+ For fixed-output derivations, each hash in the map is not the
+ corresponding output's content hash, but a hash of that hash along
+ with other constant data. The key point is that the value is a pure
+ function of the output's contents, and there are no preimage attacks
+ either spoofing an output's contents for a derivation, or
+ spoofing a derivation for an output's contents.
+
+ For regular derivations, it looks up each subderivation from its hash
+ and recurs. If the subderivation is also regular, it simply
+ substitutes the derivation path with its hash. If the subderivation
+ is fixed-output, however, it takes each output hash and pretends it
+ is a derivation hash producing a single "out" output. This is so we
+ don't leak the provenance of fixed outputs, reducing pointless cache
+ misses as the build itself won't know this.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
/* Return a fixed hash for fixed-output derivations. */
- if (drv.isFixedOutput()) {
- DerivationOutputs::const_iterator i = drv.outputs.begin();
- auto hash = std::get(i->second.output);
- return hashString(htSHA256, "fixed:out:"
- + hash.hash.printMethodAlgo() + ":"
- + hash.hash.hash.to_string(Base16, false) + ":"
- + store.printStorePath(i->second.path(store, drv.name)));
+ switch (drv.type()) {
+ case DerivationType::CAFloating:
+ throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
+ case DerivationType::CAFixed: {
+ std::map outputHashes;
+ for (const auto & i : drv.outputsAndPaths(store)) {
+ auto & dof = std::get(i.second.first.output);
+ auto hash = hashString(htSHA256, "fixed:out:"
+ + dof.hash.printMethodAlgo() + ":"
+ + dof.hash.hash.to_string(Base16, false) + ":"
+ + store.printStorePath(i.second.second));
+ outputHashes.insert_or_assign(i.first, std::move(hash));
+ }
+ return outputHashes;
+ }
+ case DerivationType::InputAddressed:
+ break;
}
/* For other derivations, replace the inputs paths with recursive
- calls to this function.*/
+ calls to this function. */
std::map inputs2;
for (auto & i : drv.inputDrvs) {
- auto h = drvHashes.find(i.first);
- if (h == drvHashes.end()) {
- assert(store.isValidPath(i.first));
- h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
- store.readDerivation(i.first), false)).first;
- }
- inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
+ const auto & res = pathDerivationModulo(store, i.first);
+ std::visit(overloaded {
+ // Regular non-CA derivation, replace derivation
+ [&](Hash drvHash) {
+ inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
+ },
+ // CA derivation's output hashes
+ [&](CaOutputHashes outputHashes) {
+ std::set justOut = { "out" };
+ for (auto & output : i.second) {
+ /* Put each one in with a single "out" output.. */
+ const auto h = outputHashes.at(output);
+ inputs2.insert_or_assign(
+ h.to_string(Base16, false),
+ justOut);
+ }
+ },
+ }, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -385,38 +511,18 @@ bool wantOutput(const string & output, const std::set & wanted)
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
- for (auto & i : outputs)
- paths.insert(i.second.path(store, name));
+ for (auto & i : outputsAndPaths(store))
+ paths.insert(i.second.second);
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
- auto hashAlgo = readString(in);
- auto hash = readString(in);
+ const auto hashAlgo = readString(in);
+ const auto hash = readString(in);
- if (hashAlgo != "") {
- auto method = FileIngestionMethod::Flat;
- if (string(hashAlgo, 0, 2) == "r:") {
- method = FileIngestionMethod::Recursive;
- hashAlgo = string(hashAlgo, 2);
- }
- auto hashType = parseHashType(hashAlgo);
- return DerivationOutput {
- .output = DerivationOutputFixed {
- .hash = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- },
- }
- };
- } else
- return DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(path),
- }
- };
+ return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
}
StringSet BasicDerivation::outputNames() const
@@ -427,6 +533,27 @@ StringSet BasicDerivation::outputNames() const
return names;
}
+DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
+ DerivationOutputsAndPaths outsAndPaths;
+ for (auto output : outputs)
+ outsAndPaths.insert(std::make_pair(
+ output.first,
+ std::make_pair(output.second, output.second.path(store, name))
+ )
+ );
+ return outsAndPaths;
+}
+
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+ DerivationOutputsAndOptPaths outsAndOptPaths;
+ for (auto output : outputs)
+ outsAndOptPaths.insert(std::make_pair(
+ output.first,
+ std::make_pair(output.second, output.second.pathOpt(store, output.first))
+ )
+ );
+ return outsAndOptPaths;
+}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
auto nameWithSuffix = drvPath.name();
@@ -467,15 +594,22 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
- for (auto & i : drv.outputs) {
+ for (auto & i : drv.outputsAndPaths(store)) {
out << i.first
- << store.printStorePath(i.second.path(store, drv.name));
- if (auto hash = std::get_if(&i.second.output)) {
- out << hash->hash.printMethodAlgo()
- << hash->hash.hash.to_string(Base16, false);
- } else {
- out << "" << "";
- }
+ << store.printStorePath(i.second.second);
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ out << "" << "";
+ },
+ [&](DerivationOutputCAFixed dof) {
+ out << dof.hash.printMethodAlgo()
+ << dof.hash.hash.to_string(Base16, false);
+ },
+ [&](DerivationOutputCAFloating dof) {
+ out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+ << "";
+ },
+ }, i.second.first.output);
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 112b2a8ca..38c4b75c0 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -6,6 +6,7 @@
#include "content-address.hh"
#include