From 2be64efb02e9eeb0fdacfec5d3314bf02ab13395 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 15 Mar 2020 02:23:17 -0400 Subject: [PATCH 001/207] Generalize `isFixedOutput` in preparation for CA drvs Today's fixed output derivations and regular derivations differ in a few ways which are largely orthogonal. This replaces `isFixedOutput` with a `type` that returns an enum of possible combinations. --- src/libstore/build.cc | 47 +++++++++++++++++++++++-------------- src/libstore/derivations.cc | 45 +++++++++++++++++++++++++++++------ src/libstore/derivations.hh | 18 +++++++++++++- src/libstore/local-store.cc | 36 +++++++++++++++++----------- 4 files changed, 106 insertions(+), 40 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9c6aedfa5..914e888b7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -795,8 +795,8 @@ private: /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; - /* Whether this is a fixed-output derivation. */ - bool fixedOutput; + /* The sort of derivation we are building. */ + DerivationType derivationType; /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; @@ -1369,12 +1369,12 @@ void DerivationGoal::inputsRealised() debug("added input paths %s", worker.store.showPaths(inputPaths)); - /* Is this a fixed-output derivation? */ - fixedOutput = drv->isFixedOutput(); + /* What type of derivation are we building? */ + derivationType = drv->type(); /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + nrRounds = DtAxisFixed & derivationType ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1724,7 +1724,7 @@ void DerivationGoal::buildDone() st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - fixedOutput || diskFull ? BuildResult::TransientFailure : + DtAxisImpure & derivationType || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1930,7 +1930,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && !noChroot; + useChroot = !(DtAxisImpure & derivationType) && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -2112,7 +2112,7 @@ void DerivationGoal::startBuilder() "nogroup:x:65534:\n") % sandboxGid).str()); /* Create /etc/hosts with localhost entry. */ - if (!fixedOutput) + if (!(DtAxisImpure & derivationType)) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -2318,7 +2318,7 @@ void DerivationGoal::startBuilder() us. */ - if (!fixedOutput) + if (!(DtAxisImpure & derivationType)) privateNetwork = true; userNamespaceSync.create(); @@ -2519,7 +2519,7 @@ void DerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + if (DtAxisFixed & derivationType) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -2530,7 +2530,7 @@ void DerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (fixedOutput) { + if (derivationType & DtAxisImpure) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -3144,7 +3144,7 @@ void DerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (fixedOutput) { + if (DtAxisImpure & derivationType) { ss.push_back("/etc/resolv.conf"); // Only use nss functions to resolve hosts and @@ -3385,7 +3385,7 @@ void DerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (fixedOutput) + if (DtAxisImpure & derivationType) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Our rwx outputs */ @@ -3644,10 +3644,10 @@ void DerivationGoal::registerOutputs() hash). */ std::string ca; - if (fixedOutput) { + if (i.second.hashAlgo != "") { - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); + bool recursive; HashType ht; + i.second.parseHashType(recursive, ht); if (!recursive) { /* The output path should be a regular file without execute permission. */ @@ -3658,11 +3658,16 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); - if (h != h2) { + // true if ither floating CA, or incorrect fixed hash. + bool needsMove = true; + + if (i.second.hash != "") { + Hash h = Hash(i.second.hash, ht); + if (h != h2) { /* Throw an error after registering the path as valid. */ @@ -3670,7 +3675,13 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + } else { + // matched the fixed hash, so no move needed. + needsMove = false; + } + } + if (needsMove) { Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); if (worker.store.isValidPath(dest)) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 205b90e55..4b72573bf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,8 +8,12 @@ namespace nix { +// Avoid shadow +HashType parseHashAlgo(const string & s) { + return parseHashType(s); +} -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashType(bool & recursive, HashType & hashType) const { recursive = false; string algo = hashAlgo; @@ -19,10 +23,16 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const algo = string(algo, 2); } - HashType hashType = parseHashType(algo); - if (hashType == htUnknown) + HashType hashType_loc = parseHashAlgo(algo); + if (hashType_loc == htUnknown) throw Error("unknown hash algorithm '%s'", algo); + hashType = hashType_loc; +} +void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +{ + HashType hashType; + parseHashType(recursive, hashType); hash = Hash(this->hash, hashType); } @@ -328,11 +338,28 @@ bool isDerivation(const string & fileName) } -bool BasicDerivation::isFixedOutput() const +DerivationType BasicDerivation::type() const { - return outputs.size() == 1 && + if (outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash != ""; + outputs.begin()->second.hash != "") + { + return DtCAFixed; + } + + auto const algo = outputs.begin()->second.hashAlgo; + if (algo != "") { + throw Error("Invalid mix of CA and regular outputs"); + } + for (auto & i : outputs) { + if (i.second.hash != "") { + throw Error("Non-fixed-output derivation has fixed output"); + } + if (i.second.hashAlgo != "") { + throw Error("Invalid mix of CA and regular outputs"); + } + } + return DtRegular; } @@ -362,13 +389,17 @@ DrvHashes drvHashes; Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ - if (drv.isFixedOutput()) { + switch (drv.type()) { + case DtCAFixed: { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + i->second.hash + ":" + store.printStorePath(i->second.path)); } + default: + break; + } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..85f52ff4c 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,6 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } + void parseHashType(bool & recursive, HashType & hashType) const; void parseHashInfo(bool & recursive, Hash & hash) const; }; @@ -33,6 +34,21 @@ typedef std::map DerivationInputs; typedef std::map StringPairs; +// Bit: +// 7: regular vs ca +// 6: floating vs fixed hash if ca, regular always floating +// 5: pure vs impure if ca, regular always pure +// _: Unassigned +enum DerivationTypeAxis : uint8_t { + DtAxisCA = 0b10000000, + DtAxisFixed = 0b01000000, + DtAxisImpure = 0b00100000, +}; +enum DerivationType : uint8_t { + DtRegular = 0b0000000, + DtCAFixed = 0b11100000, +}; + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ @@ -53,7 +69,7 @@ struct BasicDerivation bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ - bool isFixedOutput() const; + DerivationType type() const; /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd2e86f29..e048560bc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -559,21 +559,29 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); + // Don't need the answer, but do this anways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that currently prohibited. + drv.type(); - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); - - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); - } - - else { - Hash h = hashDerivationModulo(*this, drv, true); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); + std::optional h; + for (auto & i : drv.outputs) { + if (i.second.hashAlgo == "") { + if (!h) { + // somewhat expensive so we do lazily + h = hashDerivationModulo(*this, drv, true); + } + StorePath path = makeOutputPath(i.first, *h, drvName); + check(path, i.second.path, i.first); + } else { + if (i.second.hash == "") { + throw Error("Fixed output derivation needs hash"); + } + bool recursive; Hash h; + i.second.parseHashInfo(recursive, h); + StorePath path = makeFixedOutputPath(recursive, h, drvName); + check(path, i.second.path, i.first); + } } } From e5178fd22d35d58be00902b5425359b8b33019a0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Mar 2020 16:40:13 -0400 Subject: [PATCH 002/207] Fix typos Thanks @asymmetric! Co-Authored-By: asymmetric --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e048560bc..743c56cf3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -561,7 +561,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat // Don't need the answer, but do this anways to assert is proper // combination. The code below is more general and naturally allows - // combinations that currently prohibited. + // combinations that are currently prohibited. drv.type(); std::optional h; From 049179ba0776e293cd478cbb86ce7a167b64cdb0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 18 Mar 2020 19:07:05 -0400 Subject: [PATCH 003/207] Fix typos Thanks @asymmetric I failed to do them all in one batch Co-Authored-By: asymmetric --- src/libstore/build.cc | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 914e888b7..1c95038cf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3662,7 +3662,7 @@ void DerivationGoal::registerOutputs() auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); - // true if ither floating CA, or incorrect fixed hash. + // true if either floating CA, or incorrect fixed hash. bool needsMove = true; if (i.second.hash != "") { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 743c56cf3..774027a51 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -559,7 +559,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - // Don't need the answer, but do this anways to assert is proper + // Don't need the answer, but do this anyways to assert is proper // combination. The code below is more general and naturally allows // combinations that are currently prohibited. drv.type(); From f1cf3ab870343a6894c08e2bb893ea69badfc397 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Mar 2020 00:37:57 -0400 Subject: [PATCH 004/207] hashDerivationModulo: Generalize for multiple fixed ouputs per drv See documentattion in header and comments in implementation for details. This is actually done in preparation for floating ca derivations, not multi-output fixed ca derivations, but the distinction doesn't yet mattter. Thanks @cole-h for finding and fixing a bunch of typos. --- src/libexpr/primops.cc | 4 +- src/libstore/derivations.cc | 94 +++++++++++++++++++++++++------------ src/libstore/derivations.hh | 33 ++++++++++++- src/libstore/local-store.cc | 4 +- 4 files changed, 100 insertions(+), 35 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..c9a16784e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -742,7 +742,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * DerivationOutput(StorePath::dummy.clone(), "", "")); } - Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); + // Regular, non-CA derivation should always return a single hash and not + // hash per output. + Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true)); for (auto & i : outputs) { auto outPath = state.store->makeOutputPath(i, h, drvName); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 205b90e55..13f2b4770 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -338,49 +338,81 @@ bool BasicDerivation::isFixedOutput() const DrvHashes drvHashes; +/* pathDerivationModulo and hashDerivationModulo are mutually recursive + */ -/* Returns the hash of a derivation modulo fixed-output - subderivations. A fixed-output derivation is a derivation with one - output (`out') for which an expected hash and hash algorithm are - specified (using the `outputHash' and `outputHashAlgo' - attributes). We don't want changes to such derivations to - propagate upwards through the dependency graph, changing output - paths everywhere. +/* Look up the derivation by value and memoize the + `hashDerivationModulo` call. + */ +static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +{ + auto h = drvHashes.find(drvPath); + if (h == drvHashes.end()) { + assert(store.isValidPath(drvPath)); + // Cache it + h = drvHashes.insert_or_assign( + drvPath.clone(), + hashDerivationModulo( + store, + readDerivation( + store, + store.toRealPath(store.printStorePath(drvPath))), + false)).first; + } + return h->second; +} - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it - (after all, (the hash of) the file being downloaded is unchanged). - So the *output paths* should not change. On the other hand, the - *derivation paths* should change to reflect the new dependency - graph. +/* See the header for interface details. These are the implementation details. - That's what this function does: it returns a hash which is just the - hash of the derivation ATerm, except that any input derivation - paths have been replaced by the result of a recursive call to this - function, and that for fixed-output derivations we return a hash of - its output path. */ -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) + For fixed ouput derivations, each hash in the map is not the + corresponding output's content hash, but a hash of that hash along + with other constant data. The key point is that the value is a pure + function of the output's contents, and there are no preimage attacks + spoofing an either an output's contents for a derivation, or + derivation for an output's contents. + + For regular derivations, it looks up each subderivation from its hash + and recurs. If the subderivation is also regular, it simply + substitutes the derivation path with its hash. If the subderivation + is fixed-output, however, it takes each output hash and pretends it + is a derivation hash producing a single "out" output. This is so we + don't leak the provenance of fixed outputs, reducing pointless cache + misses as the build itself won't know this. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" - + store.printStorePath(i->second.path)); + std::map outputHashes; + for (const auto & i : drv.outputs) { + const Hash h = hashString(htSHA256, "fixed:out:" + + i.second.hashAlgo + ":" + + i.second.hash + ":" + + store.printStorePath(i.second.path)); + outputHashes.insert_or_assign(std::string(i.first), std::move(h)); + } + return outputHashes; } /* For other derivations, replace the inputs paths with recursive - calls to this function.*/ + calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { - auto h = drvHashes.find(i.first); - if (h == drvHashes.end()) { - assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; + const auto res = pathDerivationModulo(store, i.first); + if (const Hash *pval = std::get_if<0>(&res)) { + // regular non-CA derivation, replace derivation + inputs2.insert_or_assign(pval->to_string(Base16, false), i.second); + } else if (const std::map *pval = std::get_if<1>(&res)) { + // CA derivation's output hashes + std::set justOut = { std::string("out") }; + for (auto & output : i.second) { + /* Put each one in with a single "out" output.. */ + const auto h = pval->at(output); + inputs2.insert_or_assign( + h.to_string(Base16, false), + justOut); + } } - inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..c021bf907 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -5,6 +5,7 @@ #include "store-api.hh" #include +#include namespace nix { @@ -87,10 +88,38 @@ Derivation readDerivation(const Store & store, const Path & drvPath); // FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +typedef std::variant< + Hash, // regular DRV normalized hash + std::map // known CA drv's output hashes +> DrvHashModulo; + +/* Returns hashes with the details of fixed-output subderivations + expunged. + + A fixed-output derivation is a derivation whose outputs have a + specified content hash and hash algorithm. (Currently they must have + exactly one output (`out'), which is specified using the `outputHash' + and `outputHashAlgo' attributes, but the algorithm doesn't assume + this). We don't want changes to such derivations to propagate upwards + through the dependency graph, changing output paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it (after + all, (the hash of) the file being downloaded is unchanged). So the + *output paths* should not change. On the other hand, the *derivation + paths* should change to reflect the new dependency graph. + + For fixed output derivations, this returns a map from the names of + each output to hashes unique up to the outputs' contents. + + For regular derivations, it returns a single hash of the derivation + ATerm, after subderivations have been likewise expunged from that + derivation. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map DrvHashes; +typedef std::map DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd2e86f29..8639cbf20 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -571,7 +571,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } else { - Hash h = hashDerivationModulo(*this, drv, true); + // Regular, non-CA derivation should always return a single hash and not + // hash per output. + Hash h = std::get<0>(hashDerivationModulo(*this, drv, true)); for (auto & i : drv.outputs) check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); } From d5b3328dd1e3de8910b237d54fb9dbf36629870f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Mar 2020 23:37:52 -0400 Subject: [PATCH 005/207] Apply suggestions from code review Co-Authored-By: Cole Helbling --- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 13f2b4770..4c51bdef3 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -364,12 +364,12 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv /* See the header for interface details. These are the implementation details. - For fixed ouput derivations, each hash in the map is not the + For fixed-output derivations, each hash in the map is not the corresponding output's content hash, but a hash of that hash along with other constant data. The key point is that the value is a pure function of the output's contents, and there are no preimage attacks - spoofing an either an output's contents for a derivation, or - derivation for an output's contents. + either spoofing an output's contents for a derivation, or + spoofing a derivation for an output's contents. For regular derivations, it looks up each subderivation from its hash and recurs. If the subderivation is also regular, it simply diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c021bf907..9f8b7a23e 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -100,14 +100,14 @@ typedef std::variant< specified content hash and hash algorithm. (Currently they must have exactly one output (`out'), which is specified using the `outputHash' and `outputHashAlgo' attributes, but the algorithm doesn't assume - this). We don't want changes to such derivations to propagate upwards + this.) We don't want changes to such derivations to propagate upwards through the dependency graph, changing output paths everywhere. For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it (after - all, (the hash of) the file being downloaded is unchanged). So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. + function, we do not want to rebuild everything depending on it---after + all, (the hash of) the file being downloaded is unchanged. So the + *output paths* should not change. On the other hand, the *derivation + paths* should change to reflect the new dependency graph. For fixed output derivations, this returns a map from the names of each output to hashes unique up to the outputs' contents. From e3173242362c8421429633c0e9f64ab0211760bd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Mar 2020 23:38:51 -0400 Subject: [PATCH 006/207] Apply suggestions from code review --- src/libstore/derivations.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9f8b7a23e..5e708642e 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -109,8 +109,8 @@ typedef std::variant< *output paths* should not change. On the other hand, the *derivation paths* should change to reflect the new dependency graph. - For fixed output derivations, this returns a map from the names of - each output to hashes unique up to the outputs' contents. + For fixed-output derivations, this returns a map from the name of + each output to its hash, unique up to the output's contents. For regular derivations, it returns a single hash of the derivation ATerm, after subderivations have been likewise expunged from that From 25004030591b3d5a6ce5219f36309ac05344c92b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jun 2020 17:38:54 +0000 Subject: [PATCH 007/207] Use enum and predicates rather than bitfile for derivation type --- src/libstore/build.cc | 18 +++++++++--------- src/libstore/derivations.cc | 32 +++++++++++++++++++++++++++++--- src/libstore/derivations.hh | 29 ++++++++++++++++------------- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3c9c973d3..9dc824ecb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1382,7 +1382,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = DtAxisFixed & derivationType ? 1 : settings.buildRepeat + 1; + nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1760,7 +1760,7 @@ void DerivationGoal::buildDone() st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - DtAxisImpure & derivationType || diskFull ? BuildResult::TransientFailure : + derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1966,7 +1966,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !(DtAxisImpure & derivationType) && !noChroot; + useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -2132,7 +2132,7 @@ void DerivationGoal::startBuilder() "nogroup:x:65534:\n") % sandboxGid).str()); /* Create /etc/hosts with localhost entry. */ - if (!(DtAxisImpure & derivationType)) + if (!(derivationIsImpure(derivationType))) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -2341,7 +2341,7 @@ void DerivationGoal::startBuilder() us. */ - if (!(DtAxisImpure & derivationType)) + if (!(derivationIsImpure(derivationType))) privateNetwork = true; userNamespaceSync.create(); @@ -2542,7 +2542,7 @@ void DerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (DtAxisFixed & derivationType) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -2553,7 +2553,7 @@ void DerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (derivationType & DtAxisImpure) { + if (derivationIsImpure(derivationType)) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -3167,7 +3167,7 @@ void DerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (DtAxisImpure & derivationType) { + if (derivationIsImpure(derivationType)) { ss.push_back("/etc/resolv.conf"); // Only use nss functions to resolve hosts and @@ -3408,7 +3408,7 @@ void DerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (DtAxisImpure & derivationType) + if (derivationIsImpure(derivationType)) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Our rwx outputs */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e99515bb5..224637e64 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -37,6 +37,32 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has } +bool derivationIsCA(DerivationType dt) { + switch (dt) { + case DerivationType::Regular: return false; + case DerivationType::CAFixed: return true; + }; + // Since enums can have non-variant values, but making a `default:` would + // disable exhaustiveness warnings. + abort(); +} + +bool derivationIsFixed(DerivationType dt) { + switch (dt) { + case DerivationType::Regular: return false; + case DerivationType::CAFixed: return true; + }; + abort(); +} + +bool derivationIsImpure(DerivationType dt) { + switch (dt) { + case DerivationType::Regular: return false; + case DerivationType::CAFixed: return true; + }; + abort(); +} + BasicDerivation::BasicDerivation(const BasicDerivation & other) : platform(other.platform) , builder(other.builder) @@ -344,7 +370,7 @@ DerivationType BasicDerivation::type() const outputs.begin()->first == "out" && outputs.begin()->second.hash != "") { - return DtCAFixed; + return DerivationType::CAFixed; } auto const algo = outputs.begin()->second.hashAlgo; @@ -359,7 +385,7 @@ DerivationType BasicDerivation::type() const throw Error("Invalid mix of CA and regular outputs"); } } - return DtRegular; + return DerivationType::Regular; } @@ -390,7 +416,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput { /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { - case DtCAFixed: { + case DerivationType::CAFixed: { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 5a8d0d69c..1e0ee719d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -34,21 +34,24 @@ typedef std::map DerivationInputs; typedef std::map StringPairs; -// Bit: -// 7: regular vs ca -// 6: floating vs fixed hash if ca, regular always floating -// 5: pure vs impure if ca, regular always pure -// _: Unassigned -enum DerivationTypeAxis : uint8_t { - DtAxisCA = 0b10000000, - DtAxisFixed = 0b01000000, - DtAxisImpure = 0b00100000, -}; -enum DerivationType : uint8_t { - DtRegular = 0b0000000, - DtCAFixed = 0b11100000, +enum struct DerivationType : uint8_t { + Regular, + CAFixed, }; +/* Do the outputs of the derivation have paths calculated from their content, + or from the derivation itself? */ +bool derivationIsCA(DerivationType); + +/* Is the content of the outputs fixed a-priori via a hash? Never true for + non-CA derivations. */ +bool derivationIsFixed(DerivationType); + +/* Is the derivation impure and needs to access non-deterministic resources, or + pure and can be sandboxed? Note that whether or not we actually sandbox the + derivation is controlled separately. Never true for non-CA derivations. */ +bool derivationIsImpure(DerivationType); + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ From 3a9e4c32624b36b70cf8d553fd76a85ee97773ab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jun 2020 18:50:45 +0000 Subject: [PATCH 008/207] Don't anticipate CA but not fixed outputs for now --- src/libstore/build.cc | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9dc824ecb..0b9a022df 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3690,10 +3690,10 @@ void DerivationGoal::registerOutputs() hash). */ std::string ca; - if (i.second.hashAlgo != "") { + if (derivationIsFixed(derivationType)) { - FileIngestionMethod outputHashMode; HashType ht; - i.second.parseHashType(outputHashMode, ht); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ @@ -3706,17 +3706,12 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = outputHashMode == FileIngestionMethod::Recursive - ? hashPath(ht, actualPath).first - : hashFile(ht, actualPath); + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); - // true if either floating CA, or incorrect fixed hash. - bool needsMove = true; - - if (i.second.hash != "") { - Hash h = Hash(i.second.hash, ht); - if (h != h2) { + if (h != h2) { /* Throw an error after registering the path as valid. */ @@ -3724,13 +3719,7 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - } else { - // matched the fixed hash, so no move needed. - needsMove = false; - } - } - if (needsMove) { Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) From 74b251b2f3d6414de051c8523011c0ee3c5ea154 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jun 2020 18:53:04 +0000 Subject: [PATCH 009/207] Don't anticipate multiple CA outputs for now --- src/libstore/local-store.cc | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1db450ee8..80b48d308 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -552,29 +552,21 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); + if (derivationIsFixed(drv.type())) { + DerivationOutputs::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - std::optional h; - for (auto & i : drv.outputs) { - if (i.second.hashAlgo == "") { - if (!h) { - // somewhat expensive so we do lazily - h = hashDerivationModulo(*this, drv, true); - } - StorePath path = makeOutputPath(i.first, *h, drvName); - check(path, i.second.path, i.first); - } else { - if (i.second.hash == "") { - throw Error("Fixed output derivation needs hash"); - } - FileIngestionMethod recursive; Hash h; - i.second.parseHashInfo(recursive, h); - StorePath path = makeFixedOutputPath(recursive, h, drvName); - check(path, i.second.path, i.first); - } + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); + + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); + } + + else { + Hash h = hashDerivationModulo(*this, drv, true); + for (auto & i : drv.outputs) + check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); } } From 79c169d1c615211af69c1cbc6218f7465a4f81ed Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:49:09 -0500 Subject: [PATCH 010/207] Allow substituting from different storeDir Substituters can substitute from one store dir to another with a little bit of help. The store api just needs to have a CA so it can recompute the store path based on the new store dir. We can only do this for fixed output derivations with no references, though. --- src/libstore/local-store.cc | 16 ++++++++++++---- src/libstore/local-store.hh | 3 ++- src/libstore/misc.cc | 27 ++++++++++++++++++++++++++- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 7 +++++++ src/libstore/store-api.hh | 2 +- 7 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..6385453cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; - for (auto & path : paths) { - if (infos.count(path)) continue; + for (auto & path_ : paths) { + auto path(path_.clone()); debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); + + auto ca = pathsCA.find(printStorePath(path)); + if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) { + if (!hasPrefix(ca->second, "fixed:")) + continue; + // recompute store path so that we can use a fixed output ca + path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); + } else continue; + auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..5a976e3e1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -142,7 +142,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, + std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..1b0a055d3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath, } +static std::optional getDerivationCA(ref drv) +{ + auto outputHashMode = drv->env.find("outputHashMode"); + auto outputHashAlgo = drv->env.find("outputHashAlgo"); + auto outputHash = drv->env.find("outputHash"); + if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) { + auto ht = parseHashType(outputHashAlgo->second); + auto h = Hash(outputHash->second, ht); + FileIngestionMethod ingestionMethod; + if (outputHashMode->second == "recursive") + ingestionMethod = FileIngestionMethod::Recursive; + else if (outputHashMode->second == "flat") + ingestionMethod = FileIngestionMethod::Flat; + else + throw Error("unknown ingestion method: '%s'", outputHashMode->second); + return makeFixedOutputCA(ingestionMethod, h); + } + + return std::nullopt; +} + void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) @@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector & targets, SubstitutablePathInfos infos; StorePathSet paths; // FIXME paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + + std::map pathsCA = {}; + if (auto ca = getDerivationCA(drv)) + pathsCA.insert({outPathS, *ca}); + querySubstitutablePathInfos(paths, infos, pathsCA); if (infos.empty()) { drvState_->lock()->done = true; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..956980f49 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (paths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..867d650bc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -58,7 +58,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..40000da01 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -580,6 +580,13 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; + // recompute store path on the chance dstStore does it differently + if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { + auto info2 = make_ref(*info); + info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name()); + info = info2; + } + if (!info->narHash) { StringSink sink; srcStore->narFromPath({storePath}, sink); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..6f961329c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -445,7 +445,7 @@ public: sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos, std::map pathsCA = {}) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, From 3e3eaa90dd1aaf4684697f27726ec72aec75206c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:51:44 -0500 Subject: [PATCH 011/207] Remove hashed-mirrors --- doc/manual/command-ref/conf-file.xml | 28 ---------------------------- src/libstore/builtins/fetchurl.cc | 14 -------------- src/libstore/globals.hh | 3 --- 3 files changed, 45 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 1fa74a143..9c55526a3 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -370,34 +370,6 @@ false. - hashed-mirrors - - A list of web servers used by - builtins.fetchurl to obtain files by - hash. The default is - http://tarballs.nixos.org/. Given a hash type - ht and a base-16 hash - h, Nix will try to download the file - from - hashed-mirror/ht/h. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the default mirror - http://tarballs.nixos.org/, when building the derivation - - -builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} - - - Nix will attempt to download this file from - http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae - first. If it is not available there, if will try the original URI. - - - - http-connections The maximum number of parallel TCP connections diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 486babf14..254cb4fce 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,20 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); - } - - /* Otherwise try the specified URL. */ fetch(mainUrl); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..2e1e405b3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -339,9 +339,6 @@ public: "setuid/setgid bits or with file capabilities."}; #endif - Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - Setting minFree{this, 0, "min-free", "Automatically run the garbage collector when free disk space drops below the specified amount."}; From 11c97070f3d845a9313f137c41974c021429e834 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 10:14:03 -0500 Subject: [PATCH 012/207] Fix storeDir != storeDir condition this needs to only continue if the path replacement fails. --- src/libstore/local-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6385453cc..4cecf0992 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -866,12 +866,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto ca = pathsCA.find(printStorePath(path)); - if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) { + if (info->references.empty() && ca != pathsCA.end()) { if (!hasPrefix(ca->second, "fixed:")) continue; - // recompute store path so that we can use a fixed output ca + // recompute store path so that we can use a different store path path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); - } else continue; + } else if (sub->storeDir != storeDir) continue; auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); From 1c55f16a1648431301f044710644a674c018321d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 12:26:08 -0500 Subject: [PATCH 013/207] Add --flat to nix add-to-store This can be used to add flat hashes to the nix store. --- src/nix/add-to-store.cc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..a5a77f17c 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "flat", + .shortName = 0, + .description = "use flat file ingestion", + .handler = {&ingestionMethod, FileIngestionMethod::Flat}, + }); } std::string description() override @@ -45,10 +53,24 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + + Hash hash; + switch (ingestionMethod) { + case FileIngestionMethod::Recursive: { + hash = info.narHash; + break; + } + case FileIngestionMethod::Flat: { + HashSink hsink(htSHA256); + readFile(path, hsink); + hash = hsink.finish().first; + break; + } + } + info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) { auto source = StringSource { *sink.s }; From 2f2ac850b54110a81e3338468161cf132618c156 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:03:17 -0500 Subject: [PATCH 014/207] Compute new store path correctly --- src/libstore/local-store.cc | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4cecf0992..bf005a3b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -859,19 +859,28 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - for (auto & path_ : paths) { - auto path(path_.clone()); - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); - try { - auto info = sub->queryPathInfo(path); + for (auto & path : paths) { + auto subPath(path.clone()); - auto ca = pathsCA.find(printStorePath(path)); - if (info->references.empty() && ca != pathsCA.end()) { - if (!hasPrefix(ca->second, "fixed:")) - continue; - // recompute store path so that we can use a different store path - path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); - } else if (sub->storeDir != storeDir) continue; + auto ca_ = pathsCA.find(printStorePath(path)); + // recompute store path so that we can use a different store root + if (ca_ != pathsCA.end()) { + auto ca(ca_->second); + if (!hasPrefix(ca, "fixed:")) + continue; + FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + subPath = makeFixedOutputPath(ingestionMethod, hash, path.name()); + if (subPath != path) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !info->isContentAddressed(*sub)) + continue; auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); From 9077b9e6b28b94cdfac80ea4afbe18d13a7b2de6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:27:28 -0500 Subject: [PATCH 015/207] Separate dstStore path from srcStore path --- src/libstore/store-api.cc | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 40000da01..1b340bf6c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -649,15 +649,24 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st processGraph(pool, PathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const Path & storePathS) { + auto storePath = srcStore->parseStorePath(storePathS); + + auto info = srcStore->queryPathInfo(storePath); + auto storePathForDst = storePath.clone(); + if (hasPrefix(info->ca, "fixed:")) + { + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + } + + if (dstStore->isValidPath(storePathForDst)) { nrDone++; showProgress(); return PathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); @@ -667,9 +676,18 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st [&](const Path & storePathS) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); + auto storePath = srcStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); - if (!dstStore->isValidPath(storePath)) { + auto storePathForDst = storePath.clone(); + if (hasPrefix(info->ca, "fixed:")) + { + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + } + + if (!dstStore->isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); showProgress(); try { From 1c148a80fe05580f94e624134b923799f6cbc7b3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:39:44 -0500 Subject: [PATCH 016/207] Replace --hashed-mirrors with substituters test --- tests/check.sh | 14 +++++++------- tests/fetchurl.sh | 21 +++++++++------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index 5f25d04cb..5f4997e28 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l [ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log -path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') +path=$(nix-build check.nix -A fetchurl --no-out-link) chmod +w $path echo foo > $path chmod -w $path -nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --check # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] -nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] echo -n > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +nix-build check.nix -A hashmismatch --no-out-link echo 'Hello World' > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] # Multiple failures with --keep-going nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$? [ "$status" = "110" ] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 2535651b0..13447d774 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,7 +5,7 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -14,7 +14,7 @@ clearStore hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -25,26 +25,23 @@ hash=$(nix hash-file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) cmp $outPath fetchurl.sh -# Test the hashed mirror feature. +# Test that we can substitute from a different store dir. clearStore -hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) +other_store=file://$TEST_ROOT/other_store -mirror=$TEST_ROOT/hashed-mirror -rm -rf $mirror -mkdir -p $mirror/sha512 -ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 +hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) +storePath=$(nix add-to-store --store $other_store ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +outPath=$(nix-build -vvvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ - --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + --argstr name bla --no-out-link --substituters $other_store # Test unpacking a NAR. rm -rf $TEST_ROOT/archive From 85d01e1f658d93a0ea8d0396b1e79d03c2ff1da2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:53:30 -0500 Subject: [PATCH 017/207] =?UTF-8?q?Don=E2=80=99t=20use=20makeStorePath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1b340bf6c..afa557e16 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -583,7 +583,9 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name()); + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); info = info2; } From 5b386b05f58b1ed996afe47791d85e6d20da91a2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:57:45 -0500 Subject: [PATCH 018/207] Recompute storePath based on isContentAddressed --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index afa557e16..c8dc4744f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -581,7 +581,7 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { + if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); From e3cb536f19920018b522f76ac2661d68627b7c9e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:25:29 -0500 Subject: [PATCH 019/207] Fix add-to-store --flat to put in correct hash --- src/nix/add-to-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index a5a77f17c..26edcf95a 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -53,14 +53,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); - info.narHash = narHash; - info.narSize = sink.s->size(); - Hash hash; switch (ingestionMethod) { case FileIngestionMethod::Recursive: { - hash = info.narHash; + hash = narHash; break; } case FileIngestionMethod::Flat: { @@ -70,6 +66,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand break; } } + + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + info.narHash = narHash; + info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) { From 88120442d23a7e00833e8b0d523a6aa8072287b3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:32:52 -0500 Subject: [PATCH 020/207] Debug when storePath changes these rewrites should be transparent, but they are important to know about when debugging --- src/libstore/store-api.cc | 4 ++++ tests/fetchurl.sh | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8dc4744f..7cb48b293 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -661,6 +661,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (storePathForDst != storePath) + debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (dstStore->isValidPath(storePathForDst)) { @@ -687,6 +689,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (storePathForDst != storePath) + debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (!dstStore->isValidPath(storePathForDst)) { diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 13447d774..510f98843 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -35,12 +35,13 @@ clearStore other_store=file://$TEST_ROOT/other_store hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix add-to-store --store $other_store ./fetchurl.sh) -outPath=$(nix-build -vvvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) + +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ --argstr name bla --no-out-link --substituters $other_store # Test unpacking a NAR. From 006f1252d2fdb36c7ee66bf79f0df2ab8cd16816 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:33:24 -0500 Subject: [PATCH 021/207] Fix SRI test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can’t use custom name here because different names will have different store paths. This is a limitation of the Store API’s reliance on store paths. We might be able to get around the above in the future by using a dummy name for certain fixed output paths. --- tests/fetchurl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 510f98843..f4b86d251 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,7 +42,7 @@ outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchu # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ - --argstr name bla --no-out-link --substituters $other_store + --no-out-link --substituters $other_store # Test unpacking a NAR. rm -rf $TEST_ROOT/archive From b2cb288cdd462cbca35ffd3afda9d7ec2d595209 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 16:36:35 -0500 Subject: [PATCH 022/207] Add makeFixedOutputPathFromCA function This puts what we are already doing into a shared method. It just needs a path name and a ca and produces a store path. --- src/libstore/local-store.cc | 11 +++-------- src/libstore/store-api.cc | 31 ++++++++++++++++++------------- src/libstore/store-api.hh | 6 +++++- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bf005a3b3..9158a52e0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -862,15 +862,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, for (auto & path : paths) { auto subPath(path.clone()); - auto ca_ = pathsCA.find(printStorePath(path)); + auto ca = pathsCA.find(printStorePath(path)); // recompute store path so that we can use a different store root - if (ca_ != pathsCA.end()) { - auto ca(ca_->second); - if (!hasPrefix(ca, "fixed:")) - continue; - FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - subPath = makeFixedOutputPath(ingestionMethod, hash, path.name()); + if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { + subPath = makeFixedOutputPathFromCA(path.name(), ca->second); if (subPath != path) debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7cb48b293..bc9a6c4f5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -191,6 +191,19 @@ StorePath Store::makeFixedOutputPath( } } +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, std::string ca, + const StorePathSet & references, bool hasSelfReference) const +{ + if (hasPrefix(ca, "fixed:")) { + FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + return makeFixedOutputPath(ingestionMethod, hash, name, references, hasSelfReference); + } else if (hasPrefix(ca, "text:")) { + Hash hash(std::string(ca, 5)); + return makeTextPath(name, hash, references); + } else + throw Error("'%s' is not a valid ca", ca); +} StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const @@ -583,9 +596,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); info = info2; } @@ -656,11 +667,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath.clone(); - if (hasPrefix(info->ca, "fixed:")) - { - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (info->isContentAddressed(*srcStore)) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } @@ -684,11 +692,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath.clone(); - if (hasPrefix(info->ca, "fixed:")) - { - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (info->isContentAddressed(*srcStore)) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6f961329c..9412785b4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -352,7 +352,11 @@ public: bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, std::string ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store From c214cda9401cf50d0419038746428260b0dfdd63 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 13 Jun 2020 00:07:42 -0500 Subject: [PATCH 023/207] Correctly substitute from different storeDir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Originally, the test was only checking for different “real” storeDir. That’s an easy case to handle, but the much harder one is if different virtual store dirs are used. To do this, we need the SubstitutionGoal to know about the ca, so it can recalculate the path to copy it over. An important note here is that the store path passed to copyStorePath needs to be one for srcStore - so that queryPathInfo works properly. This also adds an error message when the store path from queryPathInfo is different from the one we requested. --- src/libstore/build.cc | 44 ++++++++++++++++++++++++++++----------- src/libstore/misc.cc | 12 +++++------ src/libstore/store-api.hh | 2 ++ tests/fetchurl.sh | 4 ++-- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..f3f6f01cc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -305,7 +305,7 @@ public: GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1195,7 +1195,7 @@ void DerivationGoal::haveDerivation() them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv))); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -4268,8 +4268,11 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* Content address for recomputing store path */ + std::optional ca; + public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut() override { abort(); }; @@ -4304,10 +4307,11 @@ public: }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(std::move(storePath)) , repair(repair) + , ca(ca) { state = &SubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); @@ -4382,14 +4386,13 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } + auto subPath = storePath.clone(); + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); try { // FIXME: make async - info = sub->queryPathInfo(storePath); + info = sub->queryPathInfo(subPath); } catch (InvalidPath &) { tryNext(); return; @@ -4408,6 +4411,19 @@ void SubstitutionGoal::tryNext() throw; } + if (info->path != storePath) { + if (info->isContentAddressed(*sub)) { + auto info2 = std::const_pointer_cast(std::shared_ptr(info)); + info2->path = storePath.clone(); + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + tryNext(); + return; + } + } + /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast(info); @@ -4493,8 +4509,12 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); + auto subPath = storePath.clone(); + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4628,11 +4648,11 @@ std::shared_ptr Worker::makeBasicDerivationGoal(StorePath && drv } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared(path.clone(), *this, repair); + goal = std::make_shared(path.clone(), *this, repair, ca); substitutionGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1b0a055d3..109e9e473 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,12 +108,12 @@ void Store::computeFSClosure(const StorePath & startPath, } -static std::optional getDerivationCA(ref drv) +std::optional getDerivationCA(const BasicDerivation & drv) { - auto outputHashMode = drv->env.find("outputHashMode"); - auto outputHashAlgo = drv->env.find("outputHashAlgo"); - auto outputHash = drv->env.find("outputHash"); - if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) { + auto outputHashMode = drv.env.find("outputHashMode"); + auto outputHashAlgo = drv.env.find("outputHashAlgo"); + auto outputHash = drv.env.find("outputHash"); + if (outputHashMode != drv.env.end() && outputHashAlgo != drv.env.end() && outputHash != drv.env.end()) { auto ht = parseHashType(outputHashAlgo->second); auto h = Hash(outputHash->second, ht); FileIngestionMethod ingestionMethod; @@ -182,7 +182,7 @@ void Store::queryMissing(const std::vector & targets, paths.insert(outPath.clone()); std::map pathsCA = {}; - if (auto ca = getDerivationCA(drv)) + if (auto ca = getDerivationCA(*drv)) pathsCA.insert({outPathS, *ca}); querySubstitutablePathInfos(paths, infos, pathsCA); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9412785b4..9918a36c6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -855,4 +855,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); +std::optional getDerivationCA(const BasicDerivation & drv); + } diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index f4b86d251..aaa38f2ed 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -32,13 +32,13 @@ cmp $outPath fetchurl.sh # Test that we can substitute from a different store dir. clearStore -other_store=file://$TEST_ROOT/other_store +other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build -vvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ From 9ac0d98a59ed042f698685234109d4b69bccf29c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 13 Jun 2020 00:14:30 -0500 Subject: [PATCH 024/207] Remove -vvvvv from tests/fetchurl.sh nix-build call --- tests/fetchurl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index aaa38f2ed..0f2044342 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -38,7 +38,7 @@ hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) -outPath=$(nix-build -vvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ From 6438ba1e990ddf76602a70d5c1143b73ed31855c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 12:31:01 -0400 Subject: [PATCH 025/207] Update strings from review comment --- src/libstore/store-api.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index bc9a6c4f5..2593b245c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -670,7 +670,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) - debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (dstStore->isValidPath(storePathForDst)) { @@ -695,7 +695,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) - debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (!dstStore->isValidPath(storePathForDst)) { diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 26edcf95a..7abb82556 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -26,7 +26,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "flat", .shortName = 0, - .description = "use flat file ingestion", + .description = "add flat file to the Nix store", .handler = {&ingestionMethod, FileIngestionMethod::Flat}, }); } From 8974755d1958824e732640f8131b0ed22ebd703b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 14:04:46 -0400 Subject: [PATCH 026/207] Add assert for replaced storePath --- src/libstore/build.cc | 10 ++++++++-- src/libstore/local-store.cc | 2 ++ src/libstore/store-api.cc | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9753db7b1..bc67ae31e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4409,8 +4409,11 @@ void SubstitutionGoal::tryNext() subs.pop_front(); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } try { // FIXME: make async @@ -4535,8 +4538,11 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } copyStorePath(ref(sub), ref(worker.store.shared_from_this()), subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a80026258..d08c7dc6a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -853,6 +853,8 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, // recompute store path so that we can use a different store root if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { subPath = makeFixedOutputPathFromCA(path.name(), ca->second); + if (sub->storeDir == storeDir) + assert(subPath == path); if (subPath != path) debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a28895086..80a69717e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -598,6 +598,8 @@ void copyStorePath(ref srcStore, ref dstStore, if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(info->path == info2->path); info = info2; } @@ -670,6 +672,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto storePathForDst = storePath; if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } @@ -695,6 +699,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto storePathForDst = storePath; if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } From be50de1142b0600afebde32130e8561e28b63b41 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 14:14:22 -0400 Subject: [PATCH 027/207] Make sure references are empty for store path replacing also copy info2 instead of casting --- src/libstore/build.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bc67ae31e..63cb70ef2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4437,8 +4437,8 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub)) { - auto info2 = std::const_pointer_cast(std::shared_ptr(info)); + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d08c7dc6a..3230312c9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -863,7 +863,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, try { auto info = sub->queryPathInfo(subPath); - if (sub->storeDir != storeDir && !info->isContentAddressed(*sub)) + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) continue; auto narInfo = std::dynamic_pointer_cast( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80a69717e..c8097da52 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -595,7 +595,7 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) @@ -670,7 +670,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); @@ -697,7 +697,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); From 5e631e3304aaf00b912c12025debfd35942a0ca9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 15:03:05 -0400 Subject: [PATCH 028/207] Add StorePathCAMap for querySubstitutablePathInfos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m not 100% sure this is wanted since it kind of makes everything have to know about ca even if they don’t really want to. But it also make things easier in dealing with looking up ca. --- src/libstore/daemon.cc | 7 +++++-- src/libstore/local-store.cc | 18 ++++++++---------- src/libstore/local-store.hh | 5 ++--- src/libstore/misc.cc | 7 ++----- src/libstore/path.hh | 2 ++ src/libstore/remote-store.cc | 14 ++++++++------ src/libstore/remote-store.hh | 4 ++-- src/libstore/store-api.hh | 4 ++-- 8 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..f7dc16948 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -612,7 +612,10 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); + StorePathCAMap pathsMap = {}; + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3230312c9..3901d532a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -841,22 +841,20 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA) +void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { for (auto & path : paths) { - auto subPath(path); + auto subPath(path.first); - auto ca = pathsCA.find(printStorePath(path)); // recompute store path so that we can use a different store root - if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { - subPath = makeFixedOutputPathFromCA(path.name(), ca->second); + if (path.second && (hasPrefix(*path.second, "fixed:") || hasPrefix(*path.second, "text:"))) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); if (sub->storeDir == storeDir) - assert(subPath == path); - if (subPath != path) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); @@ -868,7 +866,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ + infos.insert_or_assign(path.first, SubstitutablePathInfo{ info->deriver, info->references, narInfo ? narInfo->fileSize : 0, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 183d27c80..ab9fbfbe4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -139,9 +139,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, - std::map pathsCA = {}) override; + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6902dfa0c..dc991e227 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -178,10 +178,7 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - std::map pathsCA = {}; - if (auto ca = getDerivationCA(*drv)) - pathsCA.insert({outPathS, *ca}); - querySubstitutablePathInfos({outPath}, infos, pathsCA); + querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -238,7 +235,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path.path}, infos); + querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index aaebd3ec3..91962d114 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -62,6 +62,8 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; +typedef std::map> StorePathCAMap; + /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 396629a32..4538ccd4f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -308,18 +308,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; @@ -329,12 +328,15 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); writeStorePaths(*this, conn->to, paths); conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 498a5c6a0..07ed2b51a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -55,8 +55,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA = {}) override; + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 65954574f..12a6b6939 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -445,8 +445,8 @@ public: /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA = {}) { return; }; + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, From fb05a6adcfe0362249bd16527a2e44ea2611e5f3 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 16 Jun 2020 12:45:36 -0400 Subject: [PATCH 029/207] Eliminate old TeeSink abstraction This was introduced in fa125b9b28bea25a4eeb4d39a71a481563127cb9, and then "reverted" in 1cf480110879ffc8aee94b4b75999da405b71d7c, except that revert left the struct around doing nothing useful. We're removing it all the way now because we want to make a new `TeeSink` complementing the already-exiting `TeeSource`, that is actually a completely different concept as far as the class hierarchy is concerned. --- src/libstore/daemon.cc | 7 ++++--- src/libstore/export-import.cc | 11 ++++++----- src/libutil/archive.hh | 7 ------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..b2e37c5b5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -721,9 +721,10 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); + TeeSource tee(from); + ParseSink sink; + parseDump(sink, tee); + saved = std::move(*tee.data); source = std::make_unique(saved); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb9da027d..0e33fb687 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -77,8 +77,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + TeeSource tee(source); + ParseSink sink; + parseDump(sink, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -94,15 +95,15 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.data); + info.narSize = tee.data->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; + auto source = StringSource { *tee.data }; addToStore(info, source, NoRepair, checkSigs, accessor); res.push_back(info.path); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe2536..32d98a610 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,13 +63,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeSink : ParseSink -{ - TeeSource source; - - TeeSink(Source & source) : source(source) { } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); From 289b9b8dcf8dc651c2d245d9328911f7addd1626 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 16 Jun 2020 15:14:11 -0400 Subject: [PATCH 030/207] Create a new TeeSink abstraction This is a bit complex because we want to expose extra functionality the wrapped class has. Perhaps there is some inheritancy trickery to do this nicer, but I don't know it, and this is the first thing we tried after a series of attempts that did build. This design is kind of like that of Rust's Writer, Reader, or Iter adapters, which impliment more traits based on what the inner type implements. --- src/libutil/compression.hh | 12 ++++++++++++ src/libutil/serialise.hh | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index dd666a4e1..1bd118b47 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -25,4 +25,16 @@ MakeError(UnknownCompressionMethod, Error); MakeError(CompressionError, Error); +template<> +struct TeeSink> : CompressionSink +{ + MAKE_TEE_SINK(ref); + void finish() override { + orig->finish(); + } + void write(const unsigned char * data, size_t len) override { + return orig->write(data, len); + } +}; + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512..88a6b7ffe 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -181,6 +181,28 @@ struct TeeSource : Source } }; +#define MAKE_TEE_SINK(T) \ + T orig; \ + ref data; \ + TeeSink(T && orig) \ + : orig(std::move(orig)), data(make_ref()) { } \ + void operator () (const unsigned char * data, size_t len) { \ + this->data->append((const char *) data, len); \ + (*this->orig)(data, len); \ + } \ + void operator () (const std::string & s) \ + { \ + *data += s; \ + (*this->orig)(s); \ + } + +template +struct TeeSink : Sink +{ + MAKE_TEE_SINK(T); +}; + + /* A reader that consumes the original Source until 'size'. */ struct SizedSource : Source { From a835c740ca67e063fe47e7c31444b7c1cac0fe81 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 16 Jun 2020 15:14:11 -0400 Subject: [PATCH 031/207] Replace `TransferItem::status` with a local variable Everywhere seems to use `getHTTPStatus` now. --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..9566e0ae9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -56,7 +56,6 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; unsigned int attempt = 0; @@ -175,6 +174,7 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + std::string status; if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString>(line, " "); From 004570a377b2df355064d48afc54375690c3cdb0 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 16 Jun 2020 15:14:11 -0400 Subject: [PATCH 032/207] Add HTTP responses to FileTransferErrors --- src/libstore/filetransfer.cc | 26 ++++++++++++++++++++------ src/libstore/filetransfer.hh | 5 +++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9566e0ae9..d89f8388f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -122,7 +122,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -152,8 +152,18 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + decompressionSink = std::make_shared>>( + ref{ decompressionSink } + ); + } + } (*decompressionSink)((unsigned char *) contents, realSize); @@ -408,16 +418,20 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr response; + if (decompressionSink) + if (auto teeSink = std::dynamic_pointer_cast>>(decompressionSink)) + response = teeSink->data; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? FileTransferError(err, + ? FileTransferError(err, response, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : FileTransferError(err, + : FileTransferError(err, response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +689,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..8e31a9e42 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,9 +103,10 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr response; // intentionally optional template - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) + FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args) + : Error(args...), error(error), response(response) { } }; From 74b219ef6e5e171c56c8ad7385969e0d0df09ed8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 14:48:45 +0000 Subject: [PATCH 033/207] Adjust FileTransferError message to use opt response --- src/libstore/filetransfer.cc | 12 ++++++++++++ src/libstore/filetransfer.hh | 7 ++++--- src/libutil/error.hh | 3 +-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d89f8388f..fa8ad33f5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -834,6 +834,18 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + if (response) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 8e31a9e42..25ade0add 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -104,10 +104,11 @@ class FileTransferError : public Error public: FileTransfer::Error error; std::shared_ptr response; // intentionally optional + template - FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args) - : Error(args...), error(error), response(response) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e6102ce1..ac9d2e494 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -173,9 +173,8 @@ public: template SysError(const Args & ... args) - :Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } From 639e20dc3ed9c5b28138285653912de78fe0507f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 17:54:16 +0000 Subject: [PATCH 034/207] Prevent '%' in URL from causing crashes We have a larger problem that passsing computed strings to the first variable argument of many exception constructors is unsafe because that first variable argument is interpreted not as a plain string, but format string, and if it contains '%' boost::format will abort, since there are no arguments to the format string. In this particular instance '%' was used as part of an escape code in a URL, which, when the download failed, caused Nix to abort displaying the `FileTransferError`. --- src/libstore/filetransfer.cc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..e795da860 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -410,16 +410,13 @@ struct curlFileTransfer : public FileTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 - ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) - + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - ) - : FileTransferError(err, - fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); + ? FileTransferError(err, "unable to %s '%s': HTTP error %d%s", + request.verb(), request.uri, httpStatus, + code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + : FileTransferError(err, "unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -675,7 +672,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else From 1b23fe4afb8ed2c41604a1ed19cf3d49c34f46d1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 19:03:10 +0000 Subject: [PATCH 035/207] Fix bugs - Bad dynamic cast target ...classic - std::shared_ptr need explicit deref --- src/libstore/filetransfer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9ae682bbb..8b66cbdad 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -420,7 +420,7 @@ struct curlFileTransfer : public FileTransfer std::shared_ptr response; if (decompressionSink) - if (auto teeSink = std::dynamic_pointer_cast>>(decompressionSink)) + if (auto teeSink = std::dynamic_pointer_cast>>(decompressionSink)) response = teeSink->data; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted @@ -837,7 +837,7 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< { const auto hf = hintfmt(args...); if (response) { - err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), response); + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); } else { err.hint = hf; } From 0c9c1b8826395a9023606cd7749aa16ef6771941 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 14:48:57 -0400 Subject: [PATCH 036/207] Return map of StorePaths in copyPaths This allows the caller to know what values were actually added to the store. --- src/libstore/store-api.cc | 12 ++++++++++-- src/libstore/store-api.hh | 17 ++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 661dea141..0458fd9a6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -638,7 +638,7 @@ void copyStorePath(ref srcStore, ref dstStore, } -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); @@ -647,7 +647,11 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st for (auto & path : storePaths) if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); - if (missing.empty()) return; + std::map pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); + + if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -677,6 +681,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } + pathsMap.insert_or_assign(storePath, storePathForDst); if (dstStore->isValidPath(storePathForDst)) { nrDone++; @@ -704,6 +709,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } + pathsMap.insert_or_assign(storePath, storePathForDst); if (!dstStore->isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); @@ -723,6 +729,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrDone++; showProgress(); }); + + return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index fd3652588..2b12879e3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -444,8 +444,9 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; @@ -743,11 +744,13 @@ void copyStorePath(ref srcStore, ref dstStore, /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From e8e1f5282ff213be010cbe45772b9e725c01f285 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 15:19:09 -0400 Subject: [PATCH 037/207] Replace error message in getDerivationCA --- src/libstore/misc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 128c3c987..a01e81b78 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -126,7 +126,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) else if (outputHashMode->second == "flat") ingestionMethod = FileIngestionMethod::Flat; else - throw Error("unknown ingestion method: '%s'", outputHashMode->second); + throw Error("unknown outputHashMode: '%s'", outputHashMode->second); return makeFixedOutputCA(ingestionMethod, h); } From 20799a5151cbb185c1772b8e5160493b2dc2d0e8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: [PATCH 038/207] WIP: Make Hash always store a valid hash type --- src/libstore/build.cc | 2 +- src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 65 +++++++++++++----------- src/libutil/hash.hh | 17 +++---- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix-store/nix-store.cc | 10 ++-- src/nix/add-to-store.cc | 4 +- src/nix/develop.cc | 2 +- 8 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 82a2ab831..86041cabd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4997,7 +4997,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a05048290..de9b6a791 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -116,7 +116,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e49eb4569..00bb999cb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,16 +16,19 @@ namespace nix { +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; + } + abort(); +} void Hash::init() { - if (!type) abort(); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; - } + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -105,18 +108,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) { - if (h.type) - return *h.type; - else - abort(); -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -137,29 +133,36 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional type) - : type(type) +Hash::Hash(std::string_view s, std::optional optType) { size_t pos = 0; bool isSRI = false; + // Find the : or - separater, and set `isSRI` to the correct value auto sep = s.find(':'); if (sep == string::npos) { sep = s.find('-'); - if (sep != string::npos) { + if (sep != string::npos) isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); } + // Parse the has type before the separater, if there was one. + std::optional optParsedType; if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) + auto hts = s.substr(0, sep); + auto optParsedType = parseHashType(hts); + if (!optParsedType) throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); - pos = sep + 1; + } + + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) { + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + } else { + this->type = optParsedType ? *optParsedType : *optType; + if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); } init(); @@ -214,7 +217,7 @@ Hash::Hash(std::string_view s, std::optional type) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) @@ -267,7 +270,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -334,7 +337,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -342,7 +345,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +354,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..b51850ccf 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -25,14 +25,11 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - std::optional type = {}; - - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ Hash(HashType type) : type(type) { init(); }; @@ -105,7 +102,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); @@ -121,9 +118,9 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..22410c44c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,7 +153,7 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + Hash hash(ht), expectedHash(ht); std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4e02aa2bf..d061317ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash && info->narHash->type == htSHA256); + cout << fmt("%s\n", info->narHash->to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash->type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash->to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..0dda2af38 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,9 +46,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + *info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8b85caf82..4aee9f202 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -139,7 +139,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) .path = shellOutPath, .hash = DerivationOutputHash { .method = FileIngestionMethod::Flat, - .hash = Hash { }, + .hash = Hash { htSHA256 }, }, }); drv.env["out"] = store->printStorePath(shellOutPath); From e7a14118df5b53b07ecf48c8fb1ac712677250b3 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 19 Jun 2020 16:50:28 -0400 Subject: [PATCH 039/207] WIP bug fixing --- src/libexpr/primops.cc | 8 ++++---- src/libexpr/primops/fetchTree.cc | 2 +- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/build.cc | 10 ++++++---- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/derivations.cc | 2 +- src/libstore/export-import.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/nar-info.hh | 2 +- src/libstore/references.cc | 6 +++--- src/libstore/references.hh | 3 +-- src/nix/make-content-addressable.cc | 2 +- src/nix/verify.cc | 8 ++++---- 14 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f5fbd3fa6..0dd5624b2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1122,7 +1122,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1153,7 +1153,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1187,7 +1187,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .nixCode = NixCode { .errPos = pos } }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1197,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + Hash expectedHash(htSHA256); for (auto & attr : *args[0]->attrs) { const string & n(attr.name); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710a..a5b836383 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -147,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..98a3eaebb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 86041cabd..80351a675 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3730,8 +3730,8 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); @@ -3777,8 +3777,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..f3827684b 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..296bfad5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -646,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..53bd4da2a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,7 +9,7 @@ namespace nix { std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..9b8cc5c3a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..171903980 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash(s); conn->from >> info->ca; info->sigs = readStrings(conn->from); } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..8ee8d1ae8 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -79,8 +79,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { RefScanSink sink; std::map backMap; @@ -114,7 +114,7 @@ PathSet scanForReferences(const string & path, hash = sink.hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..a712dceef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index d1aba08e3..c92f894f2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -88,15 +88,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (info->ca == "") - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != info->narHash) { + if (hash.first != *info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -104,7 +104,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), hash.first.to_string(Base32, true)) }); } From 507aa48739f23f9a16c8b7079bfa6fc1806be78e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: [PATCH 040/207] WIP: Make Hash always store a valid hash type --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 8 +-- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/tree-info.cc | 2 +- src/libfetchers/tree-info.hh | 2 +- src/libstore/binary-cache-store.cc | 6 +-- src/libstore/build.cc | 12 +++-- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/daemon.cc | 4 +- src/libstore/derivations.cc | 2 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/local-store.cc | 26 ++++----- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 27 +++++----- src/libstore/nar-info.hh | 2 +- src/libstore/references.cc | 8 +-- src/libstore/references.hh | 3 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 68 +++++++++++++----------- src/libutil/hash.hh | 17 +++--- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix-store/nix-store.cc | 10 ++-- src/nix/add-to-store.cc | 4 +- src/nix/develop.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- src/nix/verify.cc | 8 +-- 29 files changed, 126 insertions(+), 124 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7..2a2a0d429 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash->to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f5fbd3fa6..0dd5624b2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1122,7 +1122,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1153,7 +1153,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1187,7 +1187,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .nixCode = NixCode { .errPos = pos } }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1197,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + Hash expectedHash(htSHA256); for (auto & attr : *args[0]->attrs) { const string & n(attr.name); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710a..d23d47592 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -23,7 +23,7 @@ void emitTreeAttrs( assert(tree.info.narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI, true)); + tree.info.narHash->to_string(SRI, true)); if (input->getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); @@ -147,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index b2d8cfc8d..432aa6182 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,7 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); } } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c7347281..9d1872097 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -11,7 +11,7 @@ namespace nix::fetchers { struct TreeInfo { - Hash narHash; + std::optional narHash; std::optional revCount; std::optional lastModified; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..98a3eaebb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 82a2ab831..80351a675 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3730,8 +3730,8 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); @@ -3777,8 +3777,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; @@ -4997,7 +4999,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..f3827684b 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..296bfad5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -646,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..53bd4da2a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,7 +9,7 @@ namespace nix { std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..9b8cc5c3a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..d3880459c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash(s); conn->from >> info->ca; info->sigs = readStrings(conn->from); } @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6b55ff7c..aa2ee5b30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -897,7 +897,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash && i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1010,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1067,12 +1067,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + HashResult hash = method == FileIngestionMethod::Recursive + ? HashResult { + hashAlgo == htSHA256 ? h : hashString(htSHA256, dump), + dump.size(), + } + : hashPath(htSHA256, realPath); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1255,9 +1255,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr hashSink; if (info->ca == "" || !info->references.count(info->path)) - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1266,7 +1266,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 552970248..6036b905e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash->to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index bb4448c90..fb84b0410 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { return Hash(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } else if (name == "NarHash") narHash = parseHashField(value); else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); + if (!ca.empty()) throw corrupt(); ca = value; } @@ -76,7 +75,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -86,11 +85,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash && narHash->type == htSHA256); + res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..4733bc388 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -79,8 +79,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { RefScanSink sink; std::map backMap; @@ -112,9 +112,9 @@ PathSet scanForReferences(const string & path, found.insert(j->second); } - hash = sink.hashSink.finish(); + auto hash = sink.hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f5f2ab7fd..c206a4538 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -462,7 +462,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 982fc22b6..c74aeec48 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -430,7 +430,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash->to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -462,7 +462,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narHash", info->narHash->to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -505,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -746,7 +746,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash->to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a05048290..de9b6a791 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -116,7 +116,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e49eb4569..35a449530 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,16 +16,19 @@ namespace nix { +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; + } + abort(); +} void Hash::init() { - if (!type) abort(); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; - } + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -105,18 +108,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) { - if (h.type) - return *h.type; - else - abort(); -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -137,31 +133,39 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional type) - : type(type) +Hash::Hash(std::string_view s, std::optional optType) { size_t pos = 0; bool isSRI = false; + // Find the : or - separater, and set `isSRI` to the correct value auto sep = s.find(':'); - if (sep == string::npos) { + if (sep == std::string_view::npos) { sep = s.find('-'); - if (sep != string::npos) { + if (sep != std::string_view::npos) isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); } - if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + if (sep != std::string_view::npos) { + auto hts = s.substr(0, sep); + auto optParsedType = parseHashTypeOpt(hts); + if (!optParsedType) throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); pos = sep + 1; } + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) { + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + } else { + this->type = optParsedType ? *optParsedType : *optType; + if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); + } + init(); size_t size = s.size() - pos; @@ -214,7 +218,7 @@ Hash::Hash(std::string_view s, std::optional type) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) @@ -267,7 +271,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -334,7 +338,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -342,7 +346,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +355,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..b51850ccf 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -25,14 +25,11 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - std::optional type = {}; - - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ Hash(HashType type) : type(type) { init(); }; @@ -105,7 +102,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); @@ -121,9 +118,9 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..22410c44c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,7 +153,7 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + Hash hash(ht), expectedHash(ht); std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4e02aa2bf..d061317ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash && info->narHash->type == htSHA256); + cout << fmt("%s\n", info->narHash->to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash->type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash->to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..0dda2af38 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,9 +46,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + *info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8b85caf82..4aee9f202 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -139,7 +139,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) .path = shellOutPath, .hash = DerivationOutputHash { .method = FileIngestionMethod::Flat, - .hash = Hash { }, + .hash = Hash { htSHA256 }, }, }); drv.env["out"] = store->printStorePath(shellOutPath); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..a712dceef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index d1aba08e3..c92f894f2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -88,15 +88,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (info->ca == "") - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != info->narHash) { + if (hash.first != *info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -104,7 +104,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), hash.first.to_string(Base32, true)) }); } From f2a6cee334255ced72a70f527cf3c283b4586e42 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 18:06:19 -0400 Subject: [PATCH 041/207] Update worker protocol to support sending storepath maps We need to also send the ca to daemon in addition to the path. --- src/libstore/daemon.cc | 11 +++++++---- src/libstore/remote-store.cc | 32 ++++++++++++++++++++++++++++---- src/libstore/worker-protocol.hh | 6 +++++- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f7dc16948..c613ab47d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -609,12 +609,15 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePathInfos: { - auto paths = readStorePaths(*store, from); - logger->startWork(); SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); + if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + auto paths = readStorePaths(*store, from); + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + } else + pathsMap = readStorePathCAMap(*store, from); + logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4538ccd4f..f6158685c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,6 +38,27 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +std::map> readStorePathCAMap(const Store & store, Source & from) +{ + StorePathCAMap paths; + auto count = readNum(from); + while (count--) { + auto path = readString(from); + auto ca = readString(from); + paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt); + } + return paths; +} + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +{ + out << paths.size(); + for (auto & i : paths) { + out << store.printStorePath(i.first); + out << (i.second ? *i.second : ""); + } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -334,10 +355,13 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << wopQuerySubstitutablePathInfos; - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + writeStorePaths(*this, conn->to, paths); + } else + writeStorePathCAMap(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ac42457fc..d2944674f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x115 +#define PROTOCOL_VERSION 0x116 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -69,5 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); +std::map> readStorePathCAMap(const Store & store, Source & from); + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); + } From bf9f040112c33213910faef40077122f1932d462 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jun 2020 16:51:39 +0000 Subject: [PATCH 042/207] Tweak declaration I think this is clearer --- src/libstore/derivations.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1f64086d7..ea57be0aa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -404,7 +404,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m }, // CA derivation's output hashes [&](CaOutputHashes outputHashes) { - std::set justOut = { std::string("out") }; + std::set justOut = { "out" }; for (auto & output : i.second) { /* Put each one in with a single "out" output.. */ const auto h = outputHashes.at(output); From 3804e3df9bb479fe1d399f29d16a1aabaf352c19 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jun 2020 21:05:37 +0000 Subject: [PATCH 043/207] Don't anticipate hash algo without hash in derivation for now When we merge with master, the new lack of string types make this case impossible (after parsing). Later, when we actually implemenent CA-derivations, we'll change the types to allow that. --- src/libstore/derivations.cc | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9864cf63e..f985e7ae5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -359,21 +359,9 @@ DerivationType BasicDerivation::type() const outputs.begin()->second.hash != "") { return DerivationType::CAFixed; + } else { + return DerivationType::Regular; } - - auto const algo = outputs.begin()->second.hashAlgo; - if (algo != "") { - throw Error("Invalid mix of CA and regular outputs"); - } - for (auto & i : outputs) { - if (i.second.hash != "") { - throw Error("Non-fixed-output derivation has fixed output"); - } - if (i.second.hashAlgo != "") { - throw Error("Invalid mix of CA and regular outputs"); - } - } - return DerivationType::Regular; } From 73ac003b377aa4750de057bba2de949295524a9c Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 22 Jun 2020 10:03:19 -0400 Subject: [PATCH 044/207] More bug fixing --- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 171903980..d3880459c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6b55ff7c..e276576e1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -897,7 +897,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1010,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", From 55d4bd6e0ea23dcf55dc33a964514230ee785bbe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jun 2020 18:08:27 +0000 Subject: [PATCH 045/207] Improve content address parsing - Ensure hash is in form - and not SRI. - Better errors if something goes wrong - string_view for no coppying --- src/libstore/content-address.cc | 77 ++++++++++++++++++++------------- src/libutil/hash.cc | 4 +- src/libutil/hash.hh | 4 +- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..216d7eb03 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,3 +1,4 @@ +#include "args.hh" #include "content-address.hh" namespace nix { @@ -40,38 +41,52 @@ std::string renderContentAddress(ContentAddress ca) { } ContentAddress parseContentAddress(std::string_view rawCa) { - auto prefixSeparator = rawCa.find(':'); - if (prefixSeparator != string::npos) { - auto prefix = string(rawCa, 0, prefixSeparator); - if (prefix == "text") { - auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); - Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != htSHA256) { - throw Error("parseContentAddress: the text hash should have type SHA256"); - } - return TextHash { hash }; - } else if (prefix == "fixed") { - // This has to be an inverse of makeFixedOutputCA - auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); - if (methodAndHash.substr(0,2) == "r:") { - std::string_view hashRaw = methodAndHash.substr(2,string::npos); - return FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = Hash(string(hashRaw)), - }; - } else { - std::string_view hashRaw = methodAndHash; - return FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = Hash(string(hashRaw)), - }; - } - } else { - throw Error("parseContentAddress: format not recognized; has to be text or fixed"); + auto rest = rawCa; + + // Ensure prefix + const auto prefixSeparator = rawCa.find(':'); + if (prefixSeparator == string::npos) + throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); + auto prefix = rest.substr(0, prefixSeparator); + rest = rest.substr(prefixSeparator + 1); + + auto parseHashType_ = [&](){ + // Parse hash type + auto algoSeparator = rest.find(':'); + HashType hashType; + if (algoSeparator == string::npos) + throw UsageError("content address hash must be in form \":\", but found: %s", rest); + hashType = parseHashType(rest.substr(0, algoSeparator)); + + rest = rest.substr(algoSeparator + 1); + + return std::move(hashType); + }; + + // Switch on prefix + if (prefix == "text") { + // No parsing of the method, "text" only support flat. + HashType hashType = parseHashType_(); + if (hashType != htSHA256) + throw Error("text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + return TextHash { + .hash = Hash { rest, std::move(hashType) }, + }; + } else if (prefix == "fixed") { + // Parse method + auto method = FileIngestionMethod::Flat; + if (rest.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + rest = rest.substr(2); } - } else { - throw Error("Not a content address because it lacks an appropriate prefix"); - } + HashType hashType = parseHashType_(); + return FixedOutputHash { + .method = method, + .hash = Hash { rest, std::move(hashType) }, + }; + } else + throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index c8fcdfed0..012fa727d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -342,7 +342,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +351,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..e4abe72ce 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -121,9 +121,9 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); From ec3a85730742497b29782c0b545911eb78d3c65e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jun 2020 18:19:40 +0000 Subject: [PATCH 046/207] Fix and clean up hash parser --- src/libutil/hash.cc | 62 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index f44210737..70a18f9a0 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -133,68 +133,66 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional optType) +Hash::Hash(std::string_view original, std::optional optType) { + auto rest = original; + size_t pos = 0; bool isSRI = false; - // Find the : or - separater, and set `isSRI` to the correct value - auto sep = s.find(':'); - if (sep == std::string_view::npos) { - sep = s.find('-'); - if (sep != std::string_view::npos) - isSRI = true; - } - // Parse the has type before the separater, if there was one. std::optional optParsedType; - if (sep != std::string_view::npos) { - auto hts = s.substr(0, sep); - auto optParsedType = parseHashTypeOpt(hts); - if (!optParsedType) - throw BadHash("unknown hash type '%s'", hts); - pos = sep + 1; + { + auto sep = rest.find(':'); + if (sep == std::string_view::npos) { + sep = rest.find('-'); + if (sep != std::string_view::npos) + isSRI = true; + } + if (sep != std::string_view::npos) { + auto hashRaw = rest.substr(0, sep); + optParsedType = parseHashType(hashRaw); + rest = rest.substr(sep + 1); + } } // Either the string or user must provide the type, if they both do they // must agree. if (!optParsedType && !optType) { - throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); } else { this->type = optParsedType ? *optParsedType : *optType; if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); + throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); } init(); - size_t size = s.size() - pos; - - if (!isSRI && size == base16Len()) { + if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", s); + throw BadHash("invalid base-16 hash '%s'", original); }; for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(s[pos + i * 2]) << 4 - | parseHexDigit(s[pos + i * 2 + 1]); + parseHexDigit(rest[pos + i * 2]) << 4 + | parseHexDigit(rest[pos + i * 2 + 1]); } } - else if (!isSRI && size == base32Len()) { + else if (!isSRI && rest.size() == base32Len()) { - for (unsigned int n = 0; n < size; ++n) { - char c = s[pos + size - n - 1]; + for (unsigned int n = 0; n < rest.size(); ++n) { + char c = rest[rest.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -204,21 +202,21 @@ Hash::Hash(std::string_view s, std::optional optType) hash[i + 1] |= digit >> (8 - j); } else { if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); } } } - else if (isSRI || size == base64Len()) { - auto d = base64Decode(s.substr(pos)); + else if (isSRI || rest.size() == base64Len()) { + auto d = base64Decode(rest); if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); assert(hashSize); memcpy(hash, d.data(), hashSize); } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) From 8d51d38e4ccf2ca0fa0b0471b32531fe287e38a4 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 23 Jun 2020 17:16:20 -0400 Subject: [PATCH 047/207] Fix test suite --- src/nix/add-to-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index de68e9501..ad1f9e91f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,7 +46,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - *info.narHash = narHash; + info.narHash = narHash; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { .method = FileIngestionMethod::Recursive, From d92d4f85a5c8a2a2385c084500a8b6bd54b54e6c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 21:22:13 +0000 Subject: [PATCH 048/207] Move ValidPathInfo to its own header --- src/libstore/nar-info.cc | 1 + src/libstore/nar-info.hh | 2 +- src/libstore/path-info.hh | 99 +++++++++++++++++++++++++++++++++++++++ src/libstore/store-api.hh | 78 +----------------------------- 4 files changed, 102 insertions(+), 78 deletions(-) create mode 100644 src/libstore/path-info.hh diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 04550ed97..ef04bc859 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,5 +1,6 @@ #include "globals.hh" #include "nar-info.hh" +#include "store-api.hh" namespace nix { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..fa38ccd78 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "path-info.hh" namespace nix { diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh new file mode 100644 index 000000000..f5dee00a6 --- /dev/null +++ b/src/libstore/path-info.hh @@ -0,0 +1,99 @@ +#pragma once + +// TODO many of thes eare not needed. +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "serialise.hh" +#include "crypto.hh" +#include "lru-cache.hh" +#include "sync.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + +namespace nix { + + +class Store; + +struct ValidPathInfo +{ + StorePath path; + std::optional deriver; + Hash narHash; + StorePathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. + */ + std::optional ca; + + bool operator == (const ValidPathInfo & i) const + { + return + path == i.path + && narHash == i.narHash + && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint(const Store & store) const; + + void sign(const Store & store, const SecretKey & secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store & store) const; + + static const size_t maxSigs = std::numeric_limits::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + ValidPathInfo(const ValidPathInfo & other) = default; + + ValidPathInfo(StorePath && path) : path(std::move(path)) { }; + ValidPathInfo(const StorePath & path) : path(path) { }; + + virtual ~ValidPathInfo() { } +}; + +typedef list ValidPathInfos; +} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 25d78c297..00b9c385c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -10,6 +10,7 @@ #include "globals.hh" #include "config.hh" #include "derivations.hh" +#include "path-info.hh" #include #include @@ -111,83 +112,6 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; -struct ValidPathInfo -{ - StorePath path; - std::optional deriver; - Hash narHash; - StorePathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ - bool ultimate = false; - - StringSet sigs; // note: not necessarily verified - - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ - std::optional ca; - - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint(const Store & store) const; - - void sign(const Store & store, const SecretKey & secretKey); - - /* Return true iff the path is verifiably content-addressed. */ - bool isContentAddressed(const Store & store) const; - - static const size_t maxSigs = std::numeric_limits::max(); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ - size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - - /* Verify a single signature. */ - bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; - - Strings shortRefs() const; - - ValidPathInfo(const ValidPathInfo & other) = default; - - ValidPathInfo(StorePath && path) : path(std::move(path)) { }; - ValidPathInfo(const StorePath & path) : path(path) { }; - - virtual ~ValidPathInfo() { } -}; - -typedef list ValidPathInfos; - enum BuildMode { bmNormal, bmRepair, bmCheck }; From 7ba0fae0ddc815d7d7b2c3a9cd3d60879baeab54 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 30 Jun 2020 11:57:09 -0400 Subject: [PATCH 049/207] Create the spitPrefix function in parser.hh --- src/libutil/parser.hh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/libutil/parser.hh diff --git a/src/libutil/parser.hh b/src/libutil/parser.hh new file mode 100644 index 000000000..64689e283 --- /dev/null +++ b/src/libutil/parser.hh @@ -0,0 +1,25 @@ +#pragma once + +#include + +namespace nix { + +// If `separator` is found, we return the portion of the string before the +// separator, and modify the string argument to contain only the part after the +// separator. Otherwise, wer return `std::nullopt`, and we leave the argument +// string alone. +std::optional splitPrefix(std::string_view & string, char separator); + +std::optional splitPrefix(std::string_view & string, char separator) { + auto sepInstance = string.find(separator); + + if (sepInstance != std::string_view::npos) { + auto prefix = string.substr(0, sepInstance); + string.remove_prefix(sepInstance+1); + return prefix; + } + + return std::nullopt; +} + +} From 77b51f4598763f0b3a435ed0e4ce98178d9e99da Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 30 Jun 2020 11:57:46 -0400 Subject: [PATCH 050/207] Factor the prefix splitting in content-address --- src/libstore/content-address.cc | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 4599cd924..8152e5215 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,5 +1,6 @@ #include "args.hh" #include "content-address.hh" +#include "parser.hh" namespace nix { @@ -43,23 +44,19 @@ std::string renderContentAddress(ContentAddress ca) { ContentAddress parseContentAddress(std::string_view rawCa) { auto rest = rawCa; - // Ensure prefix - const auto prefixSeparator = rawCa.find(':'); - if (prefixSeparator == string::npos) - throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); - auto prefix = rest.substr(0, prefixSeparator); - rest = rest.substr(prefixSeparator + 1); + std::string_view prefix; + { + auto optPrefix = splitPrefix(rest, ':'); + if (!optPrefix) + throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); + prefix = *optPrefix; + } auto parseHashType_ = [&](){ - // Parse hash type - auto algoSeparator = rest.find(':'); - HashType hashType; - if (algoSeparator == string::npos) - throw UsageError("content address hash must be in form \":\", but found: %s", rest); - hashType = parseHashType(rest.substr(0, algoSeparator)); - - rest = rest.substr(algoSeparator + 1); - + auto hashTypeRaw = splitPrefix(rest, ':'); + if (!hashTypeRaw) + throw UsageError("content address hash must be in form \":\", but found: %s", rawCa); + HashType hashType = parseHashType(*hashTypeRaw); return std::move(hashType); }; From a1f66d1d9e70d4204a3686002b02277465a6b7ab Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 30 Jun 2020 12:49:00 -0400 Subject: [PATCH 051/207] Factor the prefix splitting in hash --- src/libutil/hash.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e060700d9..26ab5a110 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -7,6 +7,7 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "parser.hh" #include "util.hh" #include "istringstream_nocopy.hh" @@ -144,17 +145,15 @@ Hash::Hash(std::string_view original, std::optional optType) // Parse the has type before the separater, if there was one. std::optional optParsedType; { - auto sep = rest.find(':'); - if (sep == std::string_view::npos) { - sep = rest.find('-'); - if (sep != std::string_view::npos) + auto hashRaw = splitPrefix(rest, ':'); + + if (!hashRaw) { + hashRaw = splitPrefix(rest, '-'); + if (hashRaw) isSRI = true; } - if (sep != std::string_view::npos) { - auto hashRaw = rest.substr(0, sep); - optParsedType = parseHashType(hashRaw); - rest = rest.substr(sep + 1); - } + if (hashRaw) + optParsedType = parseHashType(*hashRaw); } // Either the string or user must provide the type, if they both do they From b798efb829415eb47a532e9479523afdff74eca7 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 30 Jun 2020 14:10:30 -0400 Subject: [PATCH 052/207] WIP initial design --- src/libutil/hash.cc | 17 ++++++++++++----- src/libutil/hash.hh | 7 +++++-- src/libutil/parser.hh | 5 ++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 26ab5a110..36de293bb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -132,10 +132,13 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } -Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } -Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } +Hash fromSRI(std::string_view original) { -Hash::Hash(std::string_view original, std::optional optType) +} + +Hash::Hash(std::string_view s) : Hash(s, std::nullopt) { } + +static HashType newFunction(std::string_view & rest, std::optional optType) { auto rest = original; @@ -161,13 +164,17 @@ Hash::Hash(std::string_view original, std::optional optType) if (!optParsedType && !optType) { throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); } else { - this->type = optParsedType ? *optParsedType : *optType; if (optParsedType && optType && *optParsedType != *optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + return optParsedType ? *optParsedType : *optType; } +} - init(); +// mutates the string_view +Hash::Hash(std::string_view original, std::optional optType) + : Hash(original, newFunction(original, optType)) +Hash::Hash(std::string_view original, HashType type) : Hash(type) { if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index a55295912..42bd585a2 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,13 +40,16 @@ struct Hash is not present, then the hash type must be specified in the string. */ Hash(std::string_view s, std::optional type); - // type must be provided - Hash(std::string_view s, HashType type); // hash type must be part of string Hash(std::string_view s); +private: + // type must be provided, s must not include prefix + Hash(std::string_view s, HashType type); + void init(); +public: /* Check whether a hash is set. */ operator bool () const { return (bool) type; } diff --git a/src/libutil/parser.hh b/src/libutil/parser.hh index 64689e283..d3bfafe75 100644 --- a/src/libutil/parser.hh +++ b/src/libutil/parser.hh @@ -1,6 +1,7 @@ #pragma once #include +#include namespace nix { @@ -8,9 +9,7 @@ namespace nix { // separator, and modify the string argument to contain only the part after the // separator. Otherwise, wer return `std::nullopt`, and we leave the argument // string alone. -std::optional splitPrefix(std::string_view & string, char separator); - -std::optional splitPrefix(std::string_view & string, char separator) { +static inline std::optional splitPrefix(std::string_view & string, char separator) { auto sepInstance = string.find(separator); if (sepInstance != std::string_view::npos) { From c2e7f7a7129fc5366a4cad337fcd6ae319a58ce5 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 17:32:06 -0400 Subject: [PATCH 053/207] Fixed build, we still have test errors --- src/libutil/hash.cc | 46 +++++++++++++++++++++++++++++---------------- src/libutil/hash.hh | 4 +++- src/libutil/util.cc | 2 +- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 36de293bb..e8d290d13 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -132,17 +132,24 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } -Hash fromSRI(std::string_view original) { +Hash Hash::fromSRI(std::string_view original) { + auto rest = original; + // Parse the has type before the separater, if there was one. + auto hashRaw = splitPrefix(rest, '-'); + if (!hashRaw) + throw BadHash("hash '%s' is not SRI", original); + HashType parsedType = parseHashType(*hashRaw); + + return Hash(rest, std::make_pair(parsedType, true)); } -Hash::Hash(std::string_view s) : Hash(s, std::nullopt) { } +Hash::Hash(std::string_view s) : Hash(s, std::nullopt) {} -static HashType newFunction(std::string_view & rest, std::optional optType) +static std::pair newFunction(std::string_view & original, std::optional optType) { auto rest = original; - size_t pos = 0; bool isSRI = false; // Parse the has type before the separater, if there was one. @@ -166,16 +173,23 @@ static HashType newFunction(std::string_view & rest, std::optional opt } else { if (optParsedType && optType && *optParsedType != *optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); - return optParsedType ? *optParsedType : *optType; + return { + optParsedType ? *optParsedType : *optType, + isSRI, + }; } } // mutates the string_view Hash::Hash(std::string_view original, std::optional optType) - : Hash(original, newFunction(original, optType)) + : Hash(original, newFunction(original, optType)) {} -Hash::Hash(std::string_view original, HashType type) : Hash(type) { - if (!isSRI && rest.size() == base16Len()) { +Hash::Hash(std::string_view original, std::pair typeAndSRI) + : Hash(typeAndSRI.first) +{ + auto [type, isSRI] = std::move(typeAndSRI); + + if (!isSRI && original.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; @@ -186,15 +200,15 @@ Hash::Hash(std::string_view original, HashType type) : Hash(type) { for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(rest[pos + i * 2]) << 4 - | parseHexDigit(rest[pos + i * 2 + 1]); + parseHexDigit(original[i * 2]) << 4 + | parseHexDigit(original[i * 2 + 1]); } } - else if (!isSRI && rest.size() == base32Len()) { + else if (!isSRI && original.size() == base32Len()) { - for (unsigned int n = 0; n < rest.size(); ++n) { - char c = rest[rest.size() - n - 1]; + for (unsigned int n = 0; n < original.size(); ++n) { + char c = original[original.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; @@ -214,8 +228,8 @@ Hash::Hash(std::string_view original, HashType type) : Hash(type) { } } - else if (isSRI || rest.size() == base64Len()) { - auto d = base64Decode(rest); + else if (isSRI || original.size() == base64Len()) { + auto d = base64Decode(original); if (d.size() != hashSize) throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); assert(hashSize); @@ -223,7 +237,7 @@ Hash::Hash(std::string_view original, HashType type) : Hash(type) { } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", original, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 42bd585a2..570e50dd4 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -43,9 +43,11 @@ struct Hash // hash type must be part of string Hash(std::string_view s); + Hash fromSRI(std::string_view original); + private: // type must be provided, s must not include prefix - Hash(std::string_view s, HashType type); + Hash(std::string_view s, std::pair typeAndSRI); void init(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1268b146a..ed43c403f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1433,7 +1433,7 @@ string base64Decode(std::string_view s) char digit = decode[(unsigned char) c]; if (digit == -1) - throw Error("invalid character in Base64 string"); + throw Error("invalid character in Base64 string: '%c'", c); bits += 6; d = d << 6 | digit; From 274a8136fbf3d0fffb564f33464da26aab924b60 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 17:47:15 -0400 Subject: [PATCH 054/207] Correct FIXMEs in libfetchers --- src/libfetchers/fetchers.cc | 3 +-- src/libfetchers/tarball.cc | 6 ++---- src/libutil/hash.hh | 2 +- src/libutil/util.cc | 1 + 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9174c3de4..91d0d6a1d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,8 +35,7 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs) auto res = inputScheme->inputFromAttrs(attrs2); if (res) { if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = newHashAllowEmpty(*narHash, {}); + res->narHash = Hash::fromSRI(*narHash); return res; } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f5356f0af..732fac8c3 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -242,15 +242,13 @@ struct TarballInputScheme : InputScheme auto hash = input->url.query.find("hash"); if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); + input->hash = Hash::fromSRI(hash->second); input->url.query.erase(hash); } auto narHash = input->url.query.find("narHash"); if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); + input->narHash = Hash::fromSRI(narHash->second); input->url.query.erase(narHash); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 570e50dd4..887952ce5 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -43,7 +43,7 @@ struct Hash // hash type must be part of string Hash(std::string_view s); - Hash fromSRI(std::string_view original); + static Hash fromSRI(std::string_view original); private: // type must be provided, s must not include prefix diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ed43c403f..ea28ee70d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "lazy.hh" #include "util.hh" +#include "hash.hh" #include "affinity.hh" #include "sync.hh" #include "finally.hh" From 6faeec3b2a5446e3fa511ce8cb4b1a12621c6186 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 17:50:34 -0400 Subject: [PATCH 055/207] Keep the previous name, for diffing --- src/libutil/hash.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e8d290d13..448eb25f9 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -184,36 +184,36 @@ static std::pair newFunction(std::string_view & original, std::o Hash::Hash(std::string_view original, std::optional optType) : Hash(original, newFunction(original, optType)) {} -Hash::Hash(std::string_view original, std::pair typeAndSRI) +Hash::Hash(std::string_view rest, std::pair typeAndSRI) : Hash(typeAndSRI.first) { auto [type, isSRI] = std::move(typeAndSRI); - if (!isSRI && original.size() == base16Len()) { + if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", original); + throw BadHash("invalid base-16 hash '%s'", rest); }; for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(original[i * 2]) << 4 - | parseHexDigit(original[i * 2 + 1]); + parseHexDigit(rest[i * 2]) << 4 + | parseHexDigit(rest[i * 2 + 1]); } } - else if (!isSRI && original.size() == base32Len()) { + else if (!isSRI && rest.size() == base32Len()) { - for (unsigned int n = 0; n < original.size(); ++n) { - char c = original[original.size() - n - 1]; + for (unsigned int n = 0; n < rest.size(); ++n) { + char c = rest[rest.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", original); + throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -223,21 +223,21 @@ Hash::Hash(std::string_view original, std::pair typeAndSRI) hash[i + 1] |= digit >> (8 - j); } else { if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", original); + throw BadHash("invalid base-32 hash '%s'", rest); } } } - else if (isSRI || original.size() == base64Len()) { - auto d = base64Decode(original); + else if (isSRI || rest.size() == base64Len()) { + auto d = base64Decode(rest); if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest); assert(hashSize); memcpy(hash, d.data(), hashSize); } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", original, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) From d63a5ded76079008b09256aa36ef0c222919e8fa Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 17:53:24 -0400 Subject: [PATCH 056/207] Remove unused import --- src/libutil/util.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ea28ee70d..ed43c403f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,6 +1,5 @@ #include "lazy.hh" #include "util.hh" -#include "hash.hh" #include "affinity.hh" #include "sync.hh" #include "finally.hh" From c8c4bcf90e065b47c3ee2984b1f8ff696da889af Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 18:03:22 -0400 Subject: [PATCH 057/207] Inline Hash::init() --- src/libutil/hash.cc | 2 +- src/libutil/hash.hh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 448eb25f9..2087e3464 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -27,7 +27,7 @@ static size_t regularHashSize(HashType type) { abort(); } -void Hash::init() +Hash::Hash(HashType type) : type(type) { hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 887952ce5..766009438 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -32,7 +32,7 @@ struct Hash HashType type; /* Create a zero-filled hash object. */ - Hash(HashType type) : type(type) { init(); }; + Hash(HashType type); /* Initialize the hash from a string representation, in the format "[:]" or "-" (a @@ -49,8 +49,6 @@ private: // type must be provided, s must not include prefix Hash(std::string_view s, std::pair typeAndSRI); - void init(); - public: /* Check whether a hash is set. */ operator bool () const { return (bool) type; } From 263ccdd48923b730fd7e6f687583160d7b24039b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 18:34:18 -0400 Subject: [PATCH 058/207] Rename two hash constructors to proper functions --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libfetchers/git.cc | 8 ++++---- src/libfetchers/github.cc | 8 ++++---- src/libfetchers/mercurial.cc | 6 +++--- src/libfetchers/path.cc | 4 ++-- src/libstore/content-address.cc | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 2 +- src/libutil/hash.cc | 14 ++++++++++---- src/libutil/hash.hh | 6 ++++-- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- src/nix/hash.cc | 2 +- 20 files changed, 45 insertions(+), 37 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index dd7229a3d..0421318d1 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -29,7 +29,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); + rev = Hash::parseAny(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "submodules") diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9bace8f89..236219a6f 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, *attr.pos); if (std::regex_match(value, revRegex)) - rev = Hash(value, htSHA1); + rev = Hash::parseAny(value, htSHA1); else ref = value; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 75ce5ee8b..909bac78d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -225,14 +225,14 @@ struct GitInput : Input if (isLocal) { if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + input->rev = Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); repoDir = actualUrl; } else { if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); if (!rev || rev == rev2) { input->rev = rev2; return makeResult(res->first, std::move(res->second)); @@ -301,7 +301,7 @@ struct GitInput : Input } if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + input->rev = Hash::parseAny(chomp(readFile(localRefFile)), htSHA1); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -426,7 +426,7 @@ struct GitInputScheme : InputScheme input->ref = *ref; } if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + input->rev = Hash::parseAny(*rev, htSHA1); input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0bee1d6b3..449481092 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -76,7 +76,7 @@ struct GitHubInput : Input readFile( store->toRealPath( downloadFile(store, url, "source", false).storePath))); - rev = Hash(std::string { json["sha"] }, htSHA1); + rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } @@ -140,7 +140,7 @@ struct GitHubInputScheme : InputScheme if (path.size() == 2) { } else if (path.size() == 3) { if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); + input->rev = Hash::parseAny(path[2], htSHA1); else if (std::regex_match(path[2], refRegex)) input->ref = path[2]; else @@ -152,7 +152,7 @@ struct GitHubInputScheme : InputScheme if (name == "rev") { if (input->rev) throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); + input->rev = Hash::parseAny(value, htSHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) @@ -185,7 +185,7 @@ struct GitHubInputScheme : InputScheme input->repo = getStrAttr(attrs, "repo"); input->ref = maybeGetStrAttr(attrs, "ref"); if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + input->rev = Hash::parseAny(*rev, htSHA1); return input; } }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2e0d4bf4d..29f2a9d5b 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -167,7 +167,7 @@ struct MercurialInput : Input }); if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); if (!rev || rev == rev2) { input->rev = rev2; return makeResult(res->first, std::move(res->second)); @@ -210,7 +210,7 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input->rev = Hash::parseAny(tokens[0], htSHA1); auto revCount = std::stoull(tokens[1]); input->ref = tokens[2]; @@ -293,7 +293,7 @@ struct MercurialInputScheme : InputScheme input->ref = *ref; } if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + input->rev = Hash::parseAny(*rev, htSHA1); return input; } }; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ba2cc192e..1caab4165 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -101,7 +101,7 @@ struct PathInputScheme : InputScheme for (auto & [name, value] : url.query) if (name == "rev") - input->rev = Hash(value, htSHA1); + input->rev = Hash::parseAny(value, htSHA1); else if (name == "revCount") { uint64_t revCount; if (!string2Int(value, revCount)) @@ -129,7 +129,7 @@ struct PathInputScheme : InputScheme for (auto & [name, value] : attrs) if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + input->rev = Hash::parseAny(getStrAttr(attrs, "rev"), htSHA1); else if (name == "revCount") input->revCount = getIntAttr(attrs, "revCount"); else if (name == "lastModified") diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 8152e5215..2d96fb0c0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -68,7 +68,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); return TextHash { - .hash = Hash { rest, std::move(hashType) }, + .hash = Hash::parseAny(rest, std::move(hashType)), }; } else if (prefix == "fixed") { // Parse method @@ -80,7 +80,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { HashType hashType = parseHashType_(); return FixedOutputHash { .method = method, - .hash = Hash { rest, std::move(hashType) }, + .hash = Hash::parseAny(rest, std::move(hashType)), }; } else throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0b48e04c0..cc4827e64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -706,7 +706,7 @@ static void performOp(TunnelLogger * logger, ref store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); + info.narHash = Hash::parseAny(readString(from), htSHA256); info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..30ce32354 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -118,7 +118,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream const HashType hashType = parseHashType(hashAlgo); fsh = FixedOutputHash { .method = std::move(method), - .hash = Hash(hash, hashType), + .hash = Hash::parseAny(hash, hashType), }; } @@ -416,7 +416,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashType = parseHashType(hashAlgo); fsh = FixedOutputHash { .method = std::move(method), - .hash = Hash(hash, hashType), + .hash = Hash::parseAny(hash, hashType), }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 35caf23e7..f9d95fd2b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? std::optional{} : Hash{s}; + info->narHash = s.empty() ? std::optional{} : Hash::parseAnyPrefixed(s); info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9259d8f61..b75e2bdfe 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -647,7 +647,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->id = useQueryPathInfo.getInt(0); try { - info->narHash = Hash(useQueryPathInfo.getStr(1)); + info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 9ddb9957f..92da14e23 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -193,9 +193,9 @@ public: narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) - narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4)); narInfo->fileSize = queryNAR.getInt(5); - narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) narInfo->references.insert(StorePath(r)); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index c403d4bec..2a52e4098 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -12,7 +12,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto parseHashField = [&](const string & s) { try { - return Hash(s); + return Hash::parseAnyPrefixed(s); } catch (BadHash &) { throw corrupt(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 305a47340..890d96388 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -375,7 +375,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); + info->narHash = Hash::parseAny(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4083bbe1..080ce9823 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -703,7 +703,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (hashGiven) { string s; getline(str, s); - info.narHash = Hash(s, htSHA256); + info.narHash = Hash::parseAny(s, htSHA256); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2087e3464..fcc0b9eb7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -144,7 +144,10 @@ Hash Hash::fromSRI(std::string_view original) { return Hash(rest, std::make_pair(parsedType, true)); } -Hash::Hash(std::string_view s) : Hash(s, std::nullopt) {} +Hash Hash::parseAnyPrefixed(std::string_view s) +{ + return parseAny(s, std::nullopt); +} static std::pair newFunction(std::string_view & original, std::optional optType) { @@ -181,8 +184,11 @@ static std::pair newFunction(std::string_view & original, std::o } // mutates the string_view -Hash::Hash(std::string_view original, std::optional optType) - : Hash(original, newFunction(original, optType)) {} +Hash Hash::parseAny(std::string_view original, std::optional optType) +{ + auto typeAndSRI = newFunction(original, optType); + return Hash(original, typeAndSRI); +} Hash::Hash(std::string_view rest, std::pair typeAndSRI) : Hash(typeAndSRI.first) @@ -249,7 +255,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht) warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); return h; } else - return Hash(hashStr, ht); + return Hash::parseAny(hashStr, ht); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 766009438..3e413a52c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -39,9 +39,11 @@ struct Hash Subresource Integrity hash expression). If the 'type' argument is not present, then the hash type must be specified in the string. */ - Hash(std::string_view s, std::optional type); + static Hash parseAny(std::string_view s, std::optional type); // hash type must be part of string - Hash(std::string_view s); + static Hash parseAnyPrefixed(std::string_view s); + // prefix parsed separately; non SRI hash + static Hash parseAnyUnprefixed(std::string_view s, HashType type); static Hash fromSRI(std::string_view original); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 22410c44c..f752e0448 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -156,7 +156,7 @@ static int _main(int argc, char * * argv) Hash hash(ht), expectedHash(ht); std::optional storePath; if (args.size() == 2) { - expectedHash = Hash(args[1], ht); + expectedHash = Hash::parseAny(args[1], ht); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; storePath = store->makeFixedOutputPath(recursive, expectedHash, name); if (store->isValidPath(*storePath)) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ff04cbefc..cc3b07c31 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); } @@ -950,7 +950,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto deriver = readString(in); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(in), htSHA256); + info.narHash = Hash::parseAny(readString(in), htSHA256); info.references = readStorePaths(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..fb0843ce2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -103,7 +103,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout(Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash::parseAny(s, ht).to_string(base, base == SRI)); } }; From 343d1569b193e4456e2e5365921371bfd6e37205 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 10:48:47 -0400 Subject: [PATCH 059/207] Fix test suite --- src/libutil/hash.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index fcc0b9eb7..084d24170 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -149,9 +149,9 @@ Hash Hash::parseAnyPrefixed(std::string_view s) return parseAny(s, std::nullopt); } -static std::pair newFunction(std::string_view & original, std::optional optType) +static std::pair newFunction(std::string_view & rest, std::optional optType) { - auto rest = original; + auto original = rest; bool isSRI = false; From 1f9cb06db23f357f06404969820be1d3d4118a18 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 2 Jul 2020 10:59:24 -0400 Subject: [PATCH 060/207] Try next when no ca exists and have different store dirs --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 95280ef1e..20d3c7387 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4421,6 +4421,9 @@ void SubstitutionGoal::tryNext() subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + tryNext(); + return; } try { From 27c8029573ab49ad11c069ef547e3c087c73939f Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 10:58:29 -0400 Subject: [PATCH 061/207] Inline newFunction --- src/libutil/hash.cc | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 084d24170..2908a3445 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -149,11 +149,12 @@ Hash Hash::parseAnyPrefixed(std::string_view s) return parseAny(s, std::nullopt); } -static std::pair newFunction(std::string_view & rest, std::optional optType) +Hash Hash::parseAny(std::string_view original, std::optional optType) { - auto original = rest; + auto rest = original; bool isSRI = false; + HashType hashType; // Parse the has type before the separater, if there was one. std::optional optParsedType; @@ -171,23 +172,13 @@ static std::pair newFunction(std::string_view & rest, std::optio // Either the string or user must provide the type, if they both do they // must agree. - if (!optParsedType && !optType) { + if (!optParsedType && !optType) throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); - } else { - if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); - return { - optParsedType ? *optParsedType : *optType, - isSRI, - }; - } -} + else if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); -// mutates the string_view -Hash Hash::parseAny(std::string_view original, std::optional optType) -{ - auto typeAndSRI = newFunction(original, optType); - return Hash(original, typeAndSRI); + hashType = optParsedType ? *optParsedType : *optType; + return Hash(rest, std::make_pair(hashType, isSRI)); } Hash::Hash(std::string_view rest, std::pair typeAndSRI) From f61bc45d192809a040a359b22f3dbd0722613af6 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:09:04 -0400 Subject: [PATCH 062/207] Get rid of the std::pair --- src/libutil/hash.cc | 10 ++++------ src/libutil/hash.hh | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2908a3445..2f006ce1e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -141,7 +141,7 @@ Hash Hash::fromSRI(std::string_view original) { throw BadHash("hash '%s' is not SRI", original); HashType parsedType = parseHashType(*hashRaw); - return Hash(rest, std::make_pair(parsedType, true)); + return Hash(rest, parsedType, true); } Hash Hash::parseAnyPrefixed(std::string_view s) @@ -178,14 +178,12 @@ Hash Hash::parseAny(std::string_view original, std::optional optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); hashType = optParsedType ? *optParsedType : *optType; - return Hash(rest, std::make_pair(hashType, isSRI)); + return Hash(rest, hashType, isSRI); } -Hash::Hash(std::string_view rest, std::pair typeAndSRI) - : Hash(typeAndSRI.first) +Hash::Hash(std::string_view rest, HashType type, bool isSRI) + : Hash(type) { - auto [type, isSRI] = std::move(typeAndSRI); - if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 3e413a52c..4e3591a04 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -49,7 +49,7 @@ struct Hash private: // type must be provided, s must not include prefix - Hash(std::string_view s, std::pair typeAndSRI); + Hash(std::string_view s, HashType type, bool isSRI); public: /* Check whether a hash is set. */ From 9462d8a50b5443bc2dac616f94ded9ad37020094 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:11:18 -0400 Subject: [PATCH 063/207] Rename fromSRI to parseSRI for constistency --- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 4 ++-- src/libutil/hash.cc | 2 +- src/libutil/hash.hh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 91d0d6a1d..b1782feab 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,7 +35,7 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs) auto res = inputScheme->inputFromAttrs(attrs2); if (res) { if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - res->narHash = Hash::fromSRI(*narHash); + res->narHash = Hash::parseSRI(*narHash); return res; } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 732fac8c3..6ca51269b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -242,13 +242,13 @@ struct TarballInputScheme : InputScheme auto hash = input->url.query.find("hash"); if (hash != input->url.query.end()) { - input->hash = Hash::fromSRI(hash->second); + input->hash = Hash::parseSRI(hash->second); input->url.query.erase(hash); } auto narHash = input->url.query.find("narHash"); if (narHash != input->url.query.end()) { - input->narHash = Hash::fromSRI(narHash->second); + input->narHash = Hash::parseSRI(narHash->second); input->url.query.erase(narHash); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2f006ce1e..a077d40a0 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -132,7 +132,7 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } -Hash Hash::fromSRI(std::string_view original) { +Hash Hash::parseSRI(std::string_view original) { auto rest = original; // Parse the has type before the separater, if there was one. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 4e3591a04..d321cc8e1 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -45,7 +45,7 @@ struct Hash // prefix parsed separately; non SRI hash static Hash parseAnyUnprefixed(std::string_view s, HashType type); - static Hash fromSRI(std::string_view original); + static Hash parseSRI(std::string_view original); private: // type must be provided, s must not include prefix From d2e8b9ff0e8048d7b3914811b2d4bd535feac49d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 2 Jul 2020 11:12:05 -0400 Subject: [PATCH 064/207] Store subPath in SubstitutionGoal --- src/libstore/build.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 20d3c7387..12e4b6c3b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4270,6 +4270,10 @@ private: /* The store path that should be realised through a substitute. */ StorePath storePath; + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; + /* The remaining substituters. */ std::list> subs; @@ -4416,7 +4420,6 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - auto subPath = storePath; if (ca) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) @@ -4428,7 +4431,7 @@ void SubstitutionGoal::tryNext() try { // FIXME: make async - info = sub->queryPathInfo(subPath); + info = sub->queryPathInfo(subPath ? *subPath : storePath); } catch (InvalidPath &) { tryNext(); return; @@ -4548,15 +4551,8 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - auto subPath = storePath; - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { From 36cbc74689321399aeae26ff506809b8d9b24674 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:21:00 -0400 Subject: [PATCH 065/207] Inline and simplify in parseAnyPrefixed --- src/libutil/hash.cc | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a077d40a0..75f8f319c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -144,9 +144,32 @@ Hash Hash::parseSRI(std::string_view original) { return Hash(rest, parsedType, true); } -Hash Hash::parseAnyPrefixed(std::string_view s) +Hash Hash::parseAnyPrefixed(std::string_view original) { - return parseAny(s, std::nullopt); + auto rest = original; + + bool isSRI = false; + + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + { + auto hashRaw = splitPrefix(rest, ':'); + + if (!hashRaw) { + hashRaw = splitPrefix(rest, '-'); + if (hashRaw) + isSRI = true; + } + if (hashRaw) + optParsedType = parseHashType(*hashRaw); + } + + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType) + throw BadHash("hash '%s' does not include a type.", rest); + + return Hash(rest, *optParsedType, isSRI); } Hash Hash::parseAny(std::string_view original, std::optional optType) From ea48e3a5b5f1051c251184792417326c513bf00f Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:29:33 -0400 Subject: [PATCH 066/207] Abstract common parsing functionality --- src/libutil/hash.cc | 37 +++---------------------------------- src/libutil/parser.hh | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 75f8f319c..7d6b8d96e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -147,22 +147,7 @@ Hash Hash::parseSRI(std::string_view original) { Hash Hash::parseAnyPrefixed(std::string_view original) { auto rest = original; - - bool isSRI = false; - - // Parse the has type before the separater, if there was one. - std::optional optParsedType; - { - auto hashRaw = splitPrefix(rest, ':'); - - if (!hashRaw) { - hashRaw = splitPrefix(rest, '-'); - if (hashRaw) - isSRI = true; - } - if (hashRaw) - optParsedType = parseHashType(*hashRaw); - } + auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. @@ -175,23 +160,7 @@ Hash Hash::parseAnyPrefixed(std::string_view original) Hash Hash::parseAny(std::string_view original, std::optional optType) { auto rest = original; - - bool isSRI = false; - HashType hashType; - - // Parse the has type before the separater, if there was one. - std::optional optParsedType; - { - auto hashRaw = splitPrefix(rest, ':'); - - if (!hashRaw) { - hashRaw = splitPrefix(rest, '-'); - if (hashRaw) - isSRI = true; - } - if (hashRaw) - optParsedType = parseHashType(*hashRaw); - } + auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. @@ -200,7 +169,7 @@ Hash Hash::parseAny(std::string_view original, std::optional optType) else if (optParsedType && optType && *optParsedType != *optType) throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); - hashType = optParsedType ? *optParsedType : *optType; + HashType hashType = optParsedType ? *optParsedType : *optType; return Hash(rest, hashType, isSRI); } diff --git a/src/libutil/parser.hh b/src/libutil/parser.hh index d3bfafe75..d20e4dfc6 100644 --- a/src/libutil/parser.hh +++ b/src/libutil/parser.hh @@ -21,4 +21,26 @@ static inline std::optional splitPrefix(std::string_view & str return std::nullopt; } +// Mutates the string to eliminate the prefixes when found +std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { + bool isSRI = false; + + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + { + auto hashRaw = splitPrefix(rest, ':'); + + if (!hashRaw) { + hashRaw = splitPrefix(rest, '-'); + if (hashRaw) + isSRI = true; + } + if (hashRaw) + optParsedType = parseHashType(*hashRaw); + } + + return std::make_pair(optParsedType, isSRI); +} + + } From b6b10b1d4cb1cd487bbb5d2cc063ca743ae79004 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:34:40 -0400 Subject: [PATCH 067/207] Write the implementation for parseNonSRIUnprefixed --- src/libutil/hash.cc | 5 +++++ src/libutil/hash.hh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7d6b8d96e..1150e74ed 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -173,6 +173,11 @@ Hash Hash::parseAny(std::string_view original, std::optional optType) return Hash(rest, hashType, isSRI); } +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +{ + return Hash(s, type, false); +} + Hash::Hash(std::string_view rest, HashType type, bool isSRI) : Hash(type) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index d321cc8e1..af11a028d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -43,7 +43,7 @@ struct Hash // hash type must be part of string static Hash parseAnyPrefixed(std::string_view s); // prefix parsed separately; non SRI hash - static Hash parseAnyUnprefixed(std::string_view s, HashType type); + static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); From 1fc835aa223520f37e4945fa8626a096f170b188 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 2 Jul 2020 11:57:21 -0400 Subject: [PATCH 068/207] Tighten parsing for drv files and pathinfo --- src/libstore/content-address.cc | 4 ++-- src/libstore/derivations.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 2d96fb0c0..02ab6710f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -68,7 +68,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); return TextHash { - .hash = Hash::parseAny(rest, std::move(hashType)), + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else if (prefix == "fixed") { // Parse method @@ -80,7 +80,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { HashType hashType = parseHashType_(); return FixedOutputHash { .method = method, - .hash = Hash::parseAny(rest, std::move(hashType)), + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 30ce32354..149994e3e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -118,7 +118,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream const HashType hashType = parseHashType(hashAlgo); fsh = FixedOutputHash { .method = std::move(method), - .hash = Hash::parseAny(hash, hashType), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), }; } @@ -416,7 +416,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashType = parseHashType(hashAlgo); fsh = FixedOutputHash { .method = std::move(method), - .hash = Hash::parseAny(hash, hashType), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), }; } From 1be279af2622b196cc4630c48254adc96071c7e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Jul 2020 21:44:18 +0000 Subject: [PATCH 069/207] Fix Narinfo corruption detection bug The aim of this check was just to ensure each key occurs once. --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index c403d4bec..ca471463c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -66,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!value.empty()) throw corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } From a7cd7425d9341cf8a2c3af80b84cc55e874515c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Jul 2020 23:10:11 +0000 Subject: [PATCH 070/207] Move `getParsedTypeAndSRI` to a more suitable location Also mark it static --- src/libutil/hash.cc | 21 +++++++++++++++++++++ src/libutil/parser.hh | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1150e74ed..76fa67086 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -144,6 +144,27 @@ Hash Hash::parseSRI(std::string_view original) { return Hash(rest, parsedType, true); } +// Mutates the string to eliminate the prefixes when found +static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { + bool isSRI = false; + + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + { + auto hashRaw = splitPrefix(rest, ':'); + + if (!hashRaw) { + hashRaw = splitPrefix(rest, '-'); + if (hashRaw) + isSRI = true; + } + if (hashRaw) + optParsedType = parseHashType(*hashRaw); + } + + return {optParsedType, isSRI}; +} + Hash Hash::parseAnyPrefixed(std::string_view original) { auto rest = original; diff --git a/src/libutil/parser.hh b/src/libutil/parser.hh index d20e4dfc6..a6a83ce89 100644 --- a/src/libutil/parser.hh +++ b/src/libutil/parser.hh @@ -21,26 +21,5 @@ static inline std::optional splitPrefix(std::string_view & str return std::nullopt; } -// Mutates the string to eliminate the prefixes when found -std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { - bool isSRI = false; - - // Parse the has type before the separater, if there was one. - std::optional optParsedType; - { - auto hashRaw = splitPrefix(rest, ':'); - - if (!hashRaw) { - hashRaw = splitPrefix(rest, '-'); - if (hashRaw) - isSRI = true; - } - if (hashRaw) - optParsedType = parseHashType(*hashRaw); - } - - return std::make_pair(optParsedType, isSRI); -} - } From 13796be78dfa9d3a189ea6b482659c56b1301634 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Jul 2020 23:16:57 +0000 Subject: [PATCH 071/207] Have `splitPrefix` and `splitPrefixTo` parser helpers --- src/libstore/content-address.cc | 8 +++----- src/libutil/hash.cc | 6 +++--- src/libutil/parser.hh | 10 +++++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 02ab6710f..470cc62c9 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -46,14 +46,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { std::string_view prefix; { - auto optPrefix = splitPrefix(rest, ':'); + auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); prefix = *optPrefix; } auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefix(rest, ':'); + auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form \":\", but found: %s", rawCa); HashType hashType = parseHashType(*hashTypeRaw); @@ -73,10 +73,8 @@ ContentAddress parseContentAddress(std::string_view rawCa) { } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; - if (rest.substr(0, 2) == "r:") { + if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - rest = rest.substr(2); - } HashType hashType = parseHashType_(); return FixedOutputHash { .method = method, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 76fa67086..35054462c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -136,7 +136,7 @@ Hash Hash::parseSRI(std::string_view original) { auto rest = original; // Parse the has type before the separater, if there was one. - auto hashRaw = splitPrefix(rest, '-'); + auto hashRaw = splitPrefixTo(rest, '-'); if (!hashRaw) throw BadHash("hash '%s' is not SRI", original); HashType parsedType = parseHashType(*hashRaw); @@ -151,10 +151,10 @@ static std::pair, bool> getParsedTypeAndSRI(std::string_ // Parse the has type before the separater, if there was one. std::optional optParsedType; { - auto hashRaw = splitPrefix(rest, ':'); + auto hashRaw = splitPrefixTo(rest, ':'); if (!hashRaw) { - hashRaw = splitPrefix(rest, '-'); + hashRaw = splitPrefixTo(rest, '-'); if (hashRaw) isSRI = true; } diff --git a/src/libutil/parser.hh b/src/libutil/parser.hh index a6a83ce89..d19d7d8ed 100644 --- a/src/libutil/parser.hh +++ b/src/libutil/parser.hh @@ -3,13 +3,15 @@ #include #include +#include "util.hh" + namespace nix { // If `separator` is found, we return the portion of the string before the // separator, and modify the string argument to contain only the part after the // separator. Otherwise, wer return `std::nullopt`, and we leave the argument // string alone. -static inline std::optional splitPrefix(std::string_view & string, char separator) { +static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); if (sepInstance != std::string_view::npos) { @@ -21,5 +23,11 @@ static inline std::optional splitPrefix(std::string_view & str return std::nullopt; } +static inline bool splitPrefix(std::string_view & string, std::string_view prefix) { + bool res = hasPrefix(string, prefix); + if (res) + string.remove_prefix(prefix.length()); + return res; +} } From d291be444bf6e50ba13235be975db4443b162903 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Jul 2020 14:49:22 +0000 Subject: [PATCH 072/207] Fix Perl --- perl/lib/Nix/Store.xs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 2a2a0d429..595153e36 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -285,7 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - Hash h(hash, parseHashType(algo)); + auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(method, h, name); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); From d4250fef239b28b4a098e70e1deac889616dbb9a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Jul 2020 15:17:20 +0000 Subject: [PATCH 073/207] Fix Perl, again... --- perl/lib/Nix/Store.xs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 595153e36..97cc4d94e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -224,7 +224,7 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - Hash h(s, parseHashType(algo)); + auto h = Hash::parseAny(s, parseHashType(algo)); string s = h.to_string(toBase32 ? Base32 : Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { From fd42176a21d62039580c6cf6430ee19d8d3b6e1c Mon Sep 17 00:00:00 2001 From: Daniel Fitzpatrick Date: Mon, 6 Jul 2020 21:59:18 -0500 Subject: [PATCH 074/207] Add a script to install nix on non-systemd systems. --- scripts/install-generic-multi-user.sh | 799 ++++++++++++++++++++++++++ scripts/install-nix-from-closure.sh | 14 +- 2 files changed, 809 insertions(+), 4 deletions(-) create mode 100644 scripts/install-generic-multi-user.sh diff --git a/scripts/install-generic-multi-user.sh b/scripts/install-generic-multi-user.sh new file mode 100644 index 000000000..4353e9a06 --- /dev/null +++ b/scripts/install-generic-multi-user.sh @@ -0,0 +1,799 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +# installer allows overriding build user count to speed up installation +# as creating each user takes non-trivial amount of time on macos +readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" +readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" +readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + step=$((step + 1)) + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat < $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure < /dev/null >&2; then + failure < "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + fi +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat < "$SCRATCH/nix.conf" +$NIX_EXTRA_CONF +build-users-group = $NIX_BUILD_GROUP_NAME +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # A lot of this is still usable for a non-systemd multi-user install + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then + validate_starting_assumptions + fi + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + + trap finish_success EXIT +} + + +main diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 5824c2217..906e7da9b 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -35,7 +35,7 @@ fi # Determine if we could use the multi-user installer or not if [ "$(uname -s)" = "Darwin" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 -elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then +elif [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi @@ -122,9 +122,15 @@ if [ "$(uname -s)" = "Darwin" ]; then fi if [ "$INSTALL_MODE" = "daemon" ]; then - printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' - exec "$self/install-multi-user" - exit 0 + if [ -e /run/systemd/system ]; then + printf '\e[1;31mSwitching to the Systemd Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" + exit 0 + else + printf '\e[1;31mSwitching to the Non-Systemd Daemon-based Installer\e[0m\n' + exec "$self/install-generic-multi-user" + exit 0 + fi fi if [ "$(id -u)" -eq 0 ]; then From 39859b853cef3f522c2b396cfd47f4fe184e1781 Mon Sep 17 00:00:00 2001 From: Daniel Fitzpatrick Date: Tue, 7 Jul 2020 15:08:23 -0500 Subject: [PATCH 075/207] merged systemd installer with non-systemd installer. --- scripts/install-generic-multi-user.sh | 799 -------------------------- scripts/install-multi-user.sh | 43 +- scripts/install-nix-from-closure.sh | 12 +- 3 files changed, 37 insertions(+), 817 deletions(-) delete mode 100644 scripts/install-generic-multi-user.sh diff --git a/scripts/install-generic-multi-user.sh b/scripts/install-generic-multi-user.sh deleted file mode 100644 index 4353e9a06..000000000 --- a/scripts/install-generic-multi-user.sh +++ /dev/null @@ -1,799 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - -# Sourced from: -# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh -# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 -# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh -# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh -# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a -# -# however tracking which bits came from which would be impossible. - -readonly ESC='\033[0m' -readonly BOLD='\033[1m' -readonly BLUE='\033[34m' -readonly BLUE_UL='\033[4;34m' -readonly GREEN='\033[32m' -readonly GREEN_UL='\033[4;32m' -readonly RED='\033[31m' - -# installer allows overriding build user count to speed up installation -# as creating each user takes non-trivial amount of time on macos -readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} -readonly NIX_BUILD_GROUP_ID="30000" -readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" -# Please don't change this. We don't support it, because the -# default shell profile that comes with Nix doesn't support it. -readonly NIX_ROOT="/nix" -readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} - -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") -readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" -readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" - -readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" -readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" -readonly EXTRACTED_NIX_PATH="$(dirname "$0")" - -readonly ROOT_HOME=$(echo ~root) - -if [ -t 0 ]; then - readonly IS_HEADLESS='no' -else - readonly IS_HEADLESS='yes' -fi - -headless() { - if [ "$IS_HEADLESS" = "yes" ]; then - return 0 - else - return 1 - fi -} - -contactme() { - echo "We'd love to help if you need it." - echo "" - echo "If you can, open an issue at https://github.com/nixos/nix/issues" - echo "" - echo "Or feel free to contact the team," - echo " - on IRC #nixos on irc.freenode.net" - echo " - on twitter @nixos_org" -} - -uninstall_directions() { - subheader "Uninstalling nix:" - local step=0 - - step=$((step + 1)) - - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then - step=$((step + 1)) - cat < $1" -} - -bold() { - echo "$BOLD$*$ESC" -} - -ok() { - _textout "$GREEN" "$@" -} - -warning() { - warningheader "warning!" - cat - echo "" -} - -failure() { - header "oh no!" - _textout "$RED" "$@" - echo "" - _textout "$RED" "$(contactme)" - trap finish_cleanup EXIT - exit 1 -} - -ui_confirm() { - _textout "$GREEN$GREEN_UL" "$1" - - if headless; then - echo "No TTY, assuming you would say yes :)" - return 0 - fi - - local prompt="[y/n] " - echo -n "$prompt" - while read -r y; do - if [ "$y" = "y" ]; then - echo "" - return 0 - elif [ "$y" = "n" ]; then - echo "" - return 1 - else - _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" - echo -n "$prompt" - fi - done - echo "" - return 1 -} - -__sudo() { - local expl="$1" - local cmd="$2" - shift - header "sudo execution" - - echo "I am executing:" - echo "" - printf " $ sudo %s\\n" "$cmd" - echo "" - echo "$expl" - echo "" - - return 0 -} - -_sudo() { - local expl="$1" - shift - if ! headless; then - __sudo "$expl" "$*" - fi - sudo "$@" -} - - -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { - rm -rf "$SCRATCH" -} - -function finish_fail { - finish_cleanup - - failure < /dev/null >&2; then - failure < "$SCRATCH/.nix-channels" - _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" - fi -} - -welcome_to_nix() { - ok "Welcome to the Multi-User Nix Installation" - - cat < "$SCRATCH/nix.conf" -$NIX_EXTRA_CONF -build-users-group = $NIX_BUILD_GROUP_NAME -EOF - _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf -} - -main() { - if [ "$(uname -s)" = "Darwin" ]; then - # shellcheck source=./install-darwin-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # A lot of this is still usable for a non-systemd multi-user install - else - failure "Sorry, I don't know what to do on $(uname)" - fi - - welcome_to_nix - chat_about_sudo - - if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then - validate_starting_assumptions - fi - - setup_report - - if ! ui_confirm "Ready to continue?"; then - ok "Alright, no changes have been made :)" - contactme - trap finish_cleanup EXIT - exit 1 - fi - - create_build_group - create_build_users - create_directories - place_channel_configuration - install_from_extracted_nix - - configure_shell_profile - - set +eu - . /etc/profile - set -eu - - setup_default_profile - place_nix_configuration - - trap finish_success EXIT -} - - -main diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 00c9d540b..e5cc4d7ed 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix- readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" +#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" +#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" readonly EXTRACTED_NIX_PATH="$(dirname "$0")" readonly ROOT_HOME=$(echo ~root) @@ -69,9 +71,11 @@ uninstall_directions() { subheader "Uninstalling nix:" local step=0 - if poly_service_installed_check; then + if [ -e /run/systemd/system ] && poly_service_installed_check; then step=$((step + 1)) poly_service_uninstall_directions "$step" + else + step=$((step + 1)) fi for profile_target in "${PROFILE_TARGETS[@]}"; do @@ -250,7 +254,9 @@ function finish_success { echo "But fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." fi - cat < Date: Wed, 8 Jul 2020 15:38:01 -0400 Subject: [PATCH 076/207] Add name to BasicDerivation We always have a name for BasicDerivation, since we have a derivation store path that has a name. --- src/libexpr/primops.cc | 6 ++++-- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 16 ++++++++++------ src/libstore/derivations.hh | 5 +++-- src/nix-store/nix-store.cc | 4 ++-- src/nix/repl.cc | 2 +- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dec917b38..da1937e6b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -102,9 +102,11 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + StorePath storePath = state.store->parseStorePath(path); + // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { - Derivation drv = readDerivation(*state.store, realPath); + if (state.store->isStorePath(path) && state.store->isValidPath(storePath) && isDerivation(path)) { + Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ebc4d0285..4cf67064a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -474,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, std::string(drvPath.name())); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..bdc6f4f68 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -129,9 +129,11 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } -static Derivation parseDerivation(const Store & store, const string & s) +static Derivation parseDerivation(const Store & store, const string & s, string name) { Derivation drv; + drv.name = name; + istringstream_nocopy str(s); expect(str, "Derive(["); @@ -175,10 +177,10 @@ static Derivation parseDerivation(const Store & store, const string & s) } -Derivation readDerivation(const Store & store, const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath, std::string name) { try { - return parseDerivation(store, readFile(drvPath)); + return parseDerivation(store, readFile(drvPath), name); } catch (FormatError & e) { throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } @@ -196,7 +198,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), std::string(drvPath.name())); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -369,7 +371,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } @@ -435,8 +437,10 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum(in); for (size_t n = 0; n < nr; n++) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..f9d328196 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -35,6 +35,7 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + string name; BasicDerivation() { } virtual ~BasicDerivation() { }; @@ -76,7 +77,7 @@ StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath, string name); // FIXME: remove bool isDerivation(const string & fileName); @@ -93,7 +94,7 @@ bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..9a8cf0f2f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // informational only + auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv); + readDerivation(in, *store, drv, std::string(drvPath.name())); getBuildSettings(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b..a3d85ed31 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -483,7 +483,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath); + auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); From 8e0d0689be797f9e42f9b43b06f50c1af7f20b4a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 8 Jul 2020 19:11:39 -0400 Subject: [PATCH 077/207] Only store hash of fixed derivation output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we don’t need a full storepath for a fixedoutput derivation. So just putting the ingestion method + the hash is sufficient. --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/get-drvs.cc | 2 +- src/libexpr/primops.cc | 27 ++++---- src/libstore/build.cc | 76 ++++++++++++----------- src/libstore/builtins/fetchurl.cc | 21 ++++--- src/libstore/derivations.cc | 100 +++++++++++++++++++----------- src/libstore/derivations.hh | 18 ++++-- src/libstore/local-store.cc | 11 +--- src/libstore/misc.cc | 4 +- src/nix-env/nix-env.cc | 2 +- src/nix-store/nix-store.cc | 6 +- src/nix/develop.cc | 4 +- src/nix/repl.cc | 2 +- src/nix/show-derivation.cc | 8 +-- 14 files changed, 162 insertions(+), 121 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7..6dba07549 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -304,7 +304,7 @@ SV * derivationFromPath(char * drvPath) HV * outputs = newHV(); for (auto & i : drv.outputs) - hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0); + hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path(store(), drv.name)).c_str(), 0), 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 9055f59a1..5d6e39aa0 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); - outPath = store->printStorePath(i->second.path); + outPath = store->printStorePath(i->second.path(*store, drv.name)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index da1937e6b..9b613eef9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -64,7 +64,7 @@ void EvalState::realiseContext(const PathSet & context) DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); - allowedPaths->insert(store->printStorePath(i->second.path)); + allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name))); } } } @@ -120,7 +120,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -778,11 +778,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .path = std::move(outPath), - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, + }, }); } @@ -797,8 +798,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .path = StorePath::dummy, - .hash = std::optional {}, + .output = DerivationOutputIntensional { + .path = StorePath::dummy, + }, }); } @@ -809,8 +811,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .path = std::move(outPath), - .hash = std::optional(), + .output = DerivationOutputIntensional { + .path = std::move(outPath), + }, }); } } @@ -831,7 +834,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); + state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0ef2f288f..bf7d4c2f9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store))); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation() retrySubstitution = false; for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + worker.store.addTempRoot(i.second.path(worker.store, drv->name)); /* Check what outputs paths are not already valid. */ auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); @@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure() StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure); } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + outputClosure.erase(i.second.path(worker.store, drv->name)); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i); } /* Check each path (slow!). */ @@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised() for (auto & j : i.second) { auto k = inDrv.outputs.find(j); if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); + worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths); else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", @@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild() goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) + for (auto & outPath : drv->outputPaths(worker.store)) lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { @@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = drv->outputPaths(); + missingPaths = drv->outputPaths(worker.store); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name))); + deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name))); } /* Don't do a remote build if the derivation has the attribute @@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + auto outputPaths = drv->outputPaths(worker.store); std::map hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths); } } @@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder() /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name)); /* Construct the environment passed to the builder. */ initEnv(); @@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2613,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2818,7 +2818,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); + newPaths.insert(output.second.path(*this, drv.name)); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -3580,7 +3580,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); + result.insert(drv.outputs.find(i)->second.path(store, drv.name)); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3618,7 +3618,7 @@ void DerivationGoal::registerOutputs() if (hook) { bool allValid = true; for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false; if (allValid) return; } @@ -3639,23 +3639,23 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name)); for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); + if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue; Path actualPath = path; if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); + auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name)); if (r != redirectedOutputs.end()) { auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) + && redirectedBadOutputs.count(i.second.path(worker.store, drv->name)) && pathExists(redirected)) replaceValidPath(path, redirected); if (buildMode == bmCheck) @@ -3724,7 +3724,9 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - if (i.second.hash->method == FileIngestionMethod::Flat) { + FixedOutputHash outputHash = std::get(i.second.output).hash; + + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3735,13 +3737,13 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + Hash h2 = outputHash.method == FileIngestionMethod::Recursive + ? hashPath(*outputHash.hash.type, actualPath).first + : hashFile(*outputHash.hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); - if (i.second.hash->hash != h2) { + if (outputHash.hash != h2) { /* Throw an error after registering the path as valid. */ @@ -3749,7 +3751,7 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), + outputHash.hash.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3771,7 +3773,7 @@ void DerivationGoal::registerOutputs() assert(worker.store.parseStorePath(path) == dest); ca = FixedOutputHash { - .method = i.second.hash->method, + .method = outputHash.method, .hash = h2, }; } @@ -3895,7 +3897,7 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3913,7 +3915,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix; deletePath(prev); } } @@ -4214,9 +4216,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + worker.store.isValidPath(i.second.path(worker.store, drv->name)) && + (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name))); + if (good == returnValid) result.insert(i.second.path(worker.store, drv->name)); } return result; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..017a633b5 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,16 +63,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto & output = drv.outputs.begin()->second; /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); + if (auto hash = std::get_if(&output.output)) { + if (hash->hash.method == FileIngestionMethod::Flat) { + for (auto hashedMirror : settings.hashedMirrors.get()) { + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + fetch(hashedMirror + printHashType(*hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } } + } + } /* Otherwise try the specified URL. */ fetch(mainUrl); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bdc6f4f68..ffeafea87 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,12 +8,28 @@ namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +// FIXME Put this somewhere? +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + +StorePath DerivationOutput::path(const Store & store, string drvName) const +{ + return std::visit(overloaded { + [](DerivationOutputIntensional doi) { + return doi.path; + }, + [&](DerivationOutputFixed dof) { + return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + } + }, output); +} + +const StorePath & BasicDerivation::findOutput(const Store & store, const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) throw Error("derivation has no output '%s'", id); - return i->second.path; + return i->second.path(store, name); } @@ -108,7 +124,6 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -116,16 +131,21 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputIntensional { + .path = std::move(path), + } + }; } @@ -266,10 +286,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); + if (auto hash = std::get_if(&i.second.output)) { + s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); + s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false)); + } else { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } s += ')'; } @@ -325,7 +349,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::holds_alternative(outputs.begin()->second.output); } @@ -357,10 +381,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); + auto hash = std::get(i->second.output); return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + + hash.hash.printMethodAlgo() + ":" + + hash.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(i->second.path(store, drv.name))); } /* For other derivations, replace the inputs paths with recursive @@ -394,11 +419,11 @@ bool wantOutput(const string & output, const std::set & wanted) } -StorePathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths(const Store & store) const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path(store, name)); return paths; } @@ -408,7 +433,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); auto hash = readString(in); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -416,16 +440,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputIntensional { + .path = std::move(path), + } }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; } StringSet BasicDerivation::outputNames() const @@ -469,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << drv.outputs.size(); for (auto & i : drv.outputs) { out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); + << store.printStorePath(i.second.path(store, drv.name)); + if (auto hash = std::get_if(&i.second.output)) { + out << hash->hash.printMethodAlgo() + << hash->hash.hash.to_string(Base16, false); } else { out << "" << ""; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f9d328196..d8cee0f34 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,10 +13,20 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +struct DerivationOutputIntensional { StorePath path; - std::optional hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutputFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutput +{ + std::variant output; + StorePath path(const Store & store, string drvName) const; }; typedef std::map DerivationOutputs; @@ -42,7 +52,7 @@ struct BasicDerivation /* Return the path corresponding to the output identifier `id' in the given derivation. */ - const StorePath & findOutput(const std::string & id) const; + const StorePath & findOutput(const Store & store, const std::string & id) const; bool isBuiltin() const; @@ -50,7 +60,7 @@ struct BasicDerivation bool isFixedOutput() const; /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + StorePathSet outputPaths(const Store & store) const; /* Return the output names of a derivation. */ StringSet outputNames() const; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eed225349..7ecaa8743 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); } else { Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); + check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); } } @@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (printStorePath(i.second.path)) + (printStorePath(i.second.path(*this, drv.name))) .exec(); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..c4d22a634 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector & targets, PathSet invalid; for (auto & j : drv->outputs) if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); + && !isValidPath(j.second.path(*this, drv->name))) + invalid.insert(printStorePath(j.second.path(*this, drv->name))); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d74..a42859fd0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -381,7 +381,7 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); + elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput(*state.store, "out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9a8cf0f2f..2a2f16ac1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", store2->printStorePath(path.path), j); - auto outPath = store2->printStorePath(i->second.path); + auto outPath = store2->printStorePath(i->second.path(*store, drv.name)); if (store2) { if (gcRoot == "") printGCWarning(); @@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, auto drv = store->derivationFromPath(storePath); StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path); + outputs.insert(i.second.path(*store, drv.name)); return outputs; } else return {storePath}; @@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) - cout << fmt("%1%\n", store->printStorePath(j.second.path)); + cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name))); } break; } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index eb93f56fc..929fae5c4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,9 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputIntensional { + .path = shellOutPath + } }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a3d85ed31..bd72a3b19 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -486,7 +486,7 @@ bool NixRepl::processLine(string line) auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); } } else if (command == ":i") { runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 5d77cfdca..a868023d4 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand auto outputsObj(drvObj.object("outputs")); for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash) { - outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); - outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); + outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); + if (auto hash = std::get_if(&output.second.output)) { + outputObj.attr("hashAlgo", hash->hash.printMethodAlgo()); + outputObj.attr("hash", hash->hash.hash.to_string(Base16, false)); } } } From 06a4e15478920bcf8d39c36abc1a73a1e83bc9b7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 8 Jul 2020 19:27:51 -0400 Subject: [PATCH 078/207] Fix build.cc on linux --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bf7d4c2f9..9a53d9df7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path(drv->name))); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) From a7884970c57f821e274fba20473d6fcce95ff6c8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 9 Jul 2020 11:37:18 -0400 Subject: [PATCH 079/207] Fix DerivationOutputExtensional name --- src/libexpr/primops.cc | 4 ++-- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 4 ++-- src/nix/develop.cc | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9b613eef9..bd7ec628c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -798,7 +798,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = StorePath::dummy, }, }); @@ -811,7 +811,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(outPath), }, }); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ffeafea87..86f55342d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,7 +15,7 @@ template overloaded(Ts...) -> overloaded; StorePath DerivationOutput::path(const Store & store, string drvName) const { return std::visit(overloaded { - [](DerivationOutputIntensional doi) { + [](DerivationOutputExtensional doi) { return doi.path; }, [&](DerivationOutputFixed dof) { @@ -142,7 +142,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream }; } else return DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(path), } }; @@ -450,7 +450,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) }; } else return DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(path), } }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d8cee0f34..2515188ec 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,7 +13,7 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutputIntensional +struct DerivationOutputExtensional { StorePath path; }; @@ -25,7 +25,7 @@ struct DerivationOutputFixed struct DerivationOutput { - std::variant output; + std::variant output; StorePath path(const Store & store, string drvName) const; }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 929fae5c4..e03d2e057 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputIntensional { + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputExtensional { .path = shellOutPath } }); drv.env["out"] = store->printStorePath(shellOutPath); From abea26a9683d58e0bee43d9ededeb710f3617603 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 01:57:06 +0000 Subject: [PATCH 080/207] BasicDerivation::findOutput cannot return reference anymore --- src/libstore/derivations.cc | 2 +- src/libstore/derivations.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 86f55342d..bde147502 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -24,7 +24,7 @@ StorePath DerivationOutput::path(const Store & store, string drvName) const }, output); } -const StorePath & BasicDerivation::findOutput(const Store & store, const string & id) const +const StorePath BasicDerivation::findOutput(const Store & store, const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2515188ec..4694ec0dd 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -52,7 +52,7 @@ struct BasicDerivation /* Return the path corresponding to the output identifier `id' in the given derivation. */ - const StorePath & findOutput(const Store & store, const std::string & id) const; + const StorePath findOutput(const Store & store, const std::string & id) const; bool isBuiltin() const; From 1c9bec226f9ff1a9586bfefcd0c3b66ed989b2d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 02:38:03 +0000 Subject: [PATCH 081/207] Don't improperly assume path is store path --- src/libexpr/primops.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bd7ec628c..4367f9b22 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -102,10 +102,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - StorePath storePath = state.store->parseStorePath(path); - // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(storePath) && isDerivation(path)) { + auto isValidDerivationInStore = [&]() -> std::optional { + if (!state.store->isStorePath(path)) + return std::nullopt; + auto storePath = state.store->parseStorePath(path); + if (!(state.store->isValidPath(storePath) && isDerivation(path))) + return std::nullopt; + return storePath; + }; + if (auto optStorePath = isValidDerivationInStore()) { + auto storePath = *optStorePath; Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); From 13ec627e0a46d26b5de227e193a05b51af7f55e5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 03:03:12 +0000 Subject: [PATCH 082/207] Set derivation name in dervationStrict --- src/libexpr/primops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4367f9b22..14a7deed5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -591,6 +591,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Build the derivation expression by processing the attributes. */ Derivation drv; + drv.name = drvName; PathSet context; From 5d0b75e5b6a1c70203257fecc25743e5a1e009cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:02:36 +0000 Subject: [PATCH 083/207] std::string_view for new derivation name parameters --- src/libstore/derivations.cc | 8 ++++---- src/libstore/derivations.hh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bde147502..97fd21885 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -12,7 +12,7 @@ namespace nix { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath DerivationOutput::path(const Store & store, string drvName) const +StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { return std::visit(overloaded { [](DerivationOutputExtensional doi) { @@ -149,7 +149,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } -static Derivation parseDerivation(const Store & store, const string & s, string name) +static Derivation parseDerivation(const Store & store, const string & s, std::string_view name) { Derivation drv; drv.name = name; @@ -197,7 +197,7 @@ static Derivation parseDerivation(const Store & store, const string & s, string } -Derivation readDerivation(const Store & store, const Path & drvPath, std::string name) +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name) { try { return parseDerivation(store, readFile(drvPath), name); @@ -465,7 +465,7 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { drv.name = name; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4694ec0dd..81c196ff7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -26,7 +26,7 @@ struct DerivationOutputFixed struct DerivationOutput { std::variant output; - StorePath path(const Store & store, string drvName) const; + StorePath path(const Store & store, std::string_view drvName) const; }; typedef std::map DerivationOutputs; @@ -45,7 +45,7 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; - string name; + std::string name; BasicDerivation() { } virtual ~BasicDerivation() { }; @@ -87,7 +87,7 @@ StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath, string name); +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); @@ -104,7 +104,7 @@ bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); From 18152406ce9db9491385f2e67da6f7f78e8d98d5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:26:30 +0000 Subject: [PATCH 084/207] String .drv suffix to create derivation name --- src/libexpr/primops.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 11 ++++++++++- src/libstore/derivations.hh | 2 ++ src/nix-store/nix-store.cc | 2 +- src/nix/repl.cc | 2 +- 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 14a7deed5..21bd4fb52 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -113,7 +113,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args }; if (auto optStorePath = isValidDerivationInStore()) { auto storePath = *optStorePath; - Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); + Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath)); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4cf67064a..2ff53c964 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -474,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv, std::string(drvPath.name())); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 97fd21885..a85a70c0a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -218,7 +218,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), std::string(drvPath.name())); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -465,6 +465,15 @@ StringSet BasicDerivation::outputNames() const } +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { drv.name = name; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 81c196ff7..6d3b54d3f 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -64,6 +64,8 @@ struct BasicDerivation /* Return the output names of a derivation. */ StringSet outputNames() const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 39a47e19a..9605e4b32 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv, std::string(drvPath.name())); + readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath)); getBuildSettings(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bd72a3b19..74f0b3ec8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -483,7 +483,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); + auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath))); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); From a8d4707107c39d0258fef17b531ac031dd16ec88 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:54:12 +0000 Subject: [PATCH 085/207] Undo erroneous indentation change --- src/libstore/derivations.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a85a70c0a..f41edd81d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -396,7 +396,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } From 503b4256903b2cf414a6d697d19be40db299e2c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:56:20 +0000 Subject: [PATCH 086/207] DerivationOutputExtensional -> DerivationOutputInputAddressed Thanks @regnat for the great name. --- src/libexpr/primops.cc | 4 ++-- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 4 ++-- src/nix/develop.cc | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 21bd4fb52..5a9c17814 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -806,7 +806,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = StorePath::dummy, }, }); @@ -819,7 +819,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(outPath), }, }); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f41edd81d..d267468af 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,7 +15,7 @@ template overloaded(Ts...) -> overloaded; StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { return std::visit(overloaded { - [](DerivationOutputExtensional doi) { + [](DerivationOutputInputAddressed doi) { return doi.path; }, [&](DerivationOutputFixed dof) { @@ -142,7 +142,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream }; } else return DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(path), } }; @@ -450,7 +450,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) }; } else return DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(path), } }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d3b54d3f..fd8828373 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,7 +13,7 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutputExtensional +struct DerivationOutputInputAddressed { StorePath path; }; @@ -25,7 +25,7 @@ struct DerivationOutputFixed struct DerivationOutput { - std::variant output; + std::variant output; StorePath path(const Store & store, std::string_view drvName) const; }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e03d2e057..654f62f38 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputExtensional { + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = shellOutPath } }); drv.env["out"] = store->printStorePath(shellOutPath); From 886c91dfcc462d157dc2ce5265800e98d1bc45dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 18:26:10 +0000 Subject: [PATCH 087/207] Try to fix perl bindings --- perl/lib/Nix/Store.xs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 6dba07549..522de5802 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath) HV * outputs = newHV(); for (auto & i : drv.outputs) - hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path(store(), drv.name)).c_str(), 0), 0); + hv_store( + outputs, i.first.c_str(), i.first.size(), + newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0), + 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); From ffc18583b1a086849ac0efd17da40ff510299b52 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 22:15:14 +0000 Subject: [PATCH 088/207] Move C++17 "pattern matching" boilerplat to utils.hh --- src/libstore/content-address.cc | 4 ---- src/libstore/derivations.cc | 4 ---- src/libstore/store-api.cc | 4 ---- src/libutil/util.hh | 5 +++++ 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 6cb69d0a9..f45f77d5c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -24,10 +24,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d267468af..ce2025933 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,10 +8,6 @@ namespace nix { -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { return std::visit(overloaded { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8d46bb436..62514d3be 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -802,10 +802,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - bool ValidPathInfo::isContentAddressed(const Store & store) const { if (! ca) return false; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3641daaec..d38657843 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -597,4 +597,9 @@ constexpr auto enumerate(T && iterable) } +// C++17 std::visit boilerplate +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + } From fedfc913ad75984b476e7838d6254c547f9134d5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 16:12:21 +0000 Subject: [PATCH 089/207] Use more std::visit to prepare for new variant N.B. not using `std::visit` for fetchurl because there is no attempt to handle all the cases (e.g. no `else`) and lambda complicates early return. --- src/libstore/derivations.cc | 32 +++++++++++++++++++------------- src/libstore/derivations.hh | 5 ++++- src/nix/show-derivation.cc | 13 +++++++++---- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ce2025933..09683a005 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -283,13 +283,16 @@ string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); - if (auto hash = std::get_if(&i.second.output)) { - s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); - s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false)); - } else { - s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, ""); - } + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + }, + [&](DerivationOutputFixed dof) { + s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + }, + }, i.second.output); s += ')'; } @@ -503,12 +506,15 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr for (auto & i : drv.outputs) { out << i.first << store.printStorePath(i.second.path(store, drv.name)); - if (auto hash = std::get_if(&i.second.output)) { - out << hash->hash.printMethodAlgo() - << hash->hash.hash.to_string(Base16, false); - } else { - out << "" << ""; - } + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + out << "" << ""; + }, + [&](DerivationOutputFixed dof) { + out << dof.hash.printMethodAlgo() + << dof.hash.hash.to_string(Base16, false); + }, + }, i.second.output); } writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fd8828373..4dc542536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -25,7 +25,10 @@ struct DerivationOutputFixed struct DerivationOutput { - std::variant output; + std::variant< + DerivationOutputInputAddressed, + DerivationOutputFixed + > output; StorePath path(const Store & store, std::string_view drvName) const; }; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index a868023d4..f9952e177 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -70,10 +70,15 @@ struct CmdShowDerivation : InstallablesCommand for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); - if (auto hash = std::get_if(&output.second.output)) { - outputObj.attr("hashAlgo", hash->hash.printMethodAlgo()); - outputObj.attr("hash", hash->hash.hash.to_string(Base16, false)); - } + + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + }, + [&](DerivationOutputFixed dof) { + outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); + outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); + }, + }, output.second.output); } } From 230c9b4329b3d285e57f4cce058c121256187da1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 16:12:21 +0000 Subject: [PATCH 090/207] Change types to prepare the way for CA derivations We've added the variant to `DerivationOutput` to support them, but made `DerivationOutput::path` partial to avoid actually implementing them. With this chage, we can all collaborate on "just" removing `DerivationOutput::path` calls to implement CA derivations. --- src/libstore/derivations.cc | 71 +++++++++++++++++++++++++------------ src/libstore/derivations.hh | 21 +++++++++-- src/nix/show-derivation.cc | 3 ++ 3 files changed, 71 insertions(+), 24 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 09683a005..375d089ec 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,15 +8,20 @@ namespace nix { -StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const +std::optional DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const { return std::visit(overloaded { - [](DerivationOutputInputAddressed doi) { - return doi.path; + [](DerivationOutputInputAddressed doi) -> std::optional { + return { doi.path }; + }, + [&](DerivationOutputFixed dof) -> std::optional { + return { + store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName) + }; + }, + [](DerivationOutputFloating dof) -> std::optional { + return std::nullopt; }, - [&](DerivationOutputFixed dof) { - return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - } }, output); } @@ -128,14 +133,21 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } const HashType hashType = parseHashType(hashAlgo); - return DerivationOutput { - .output = DerivationOutputFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }, - } - }; + return hash != "" + ? DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + } + : DerivationOutput { + .output = DerivationOutputFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }, + }; } else return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -292,6 +304,10 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, + [&](DerivationOutputFloating dof) { + s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, ""); + }, }, i.second.output); s += ')'; } @@ -439,14 +455,21 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } auto hashType = parseHashType(hashAlgo); - return DerivationOutput { - .output = DerivationOutputFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }, - } - }; + return hash != "" + ? DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + } + : DerivationOutput { + .output = DerivationOutputFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }, + }; } else return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -514,6 +537,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, + [&](DerivationOutputFloating dof) { + out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << ""; + }, }, i.second.output); } writeStorePaths(store, out, drv.inputSrcs); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4dc542536..36ac09210 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -15,6 +15,8 @@ namespace nix { struct DerivationOutputInputAddressed { + /* Will need to become `std::optional` once input-addressed + derivations are allowed to depend on cont-addressed derivations */ StorePath path; }; @@ -23,13 +25,28 @@ struct DerivationOutputFixed FixedOutputHash hash; /* hash used for expected hash computation */ }; +struct DerivationOutputFloating +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + struct DerivationOutput { std::variant< DerivationOutputInputAddressed, - DerivationOutputFixed + DerivationOutputFixed, + DerivationOutputFloating > output; - StorePath path(const Store & store, std::string_view drvName) const; + std::optional hashAlgoOpt(const Store & store) const; + std::optional pathOpt(const Store & store, std::string_view drvName) const; + /* DEPRECATED: Remove after CA drvs are fully implemented */ + StorePath path(const Store & store, std::string_view drvName) const { + auto p = pathOpt(store, drvName); + if (!p) throw Error("floating content-addressed derivations are not yet implemented"); + return *p; + } }; typedef std::map DerivationOutputs; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index f9952e177..95898d566 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -78,6 +78,9 @@ struct CmdShowDerivation : InstallablesCommand outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, + [&](DerivationOutputFloating dof) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + }, }, output.second.output); } } From 9ec10046e04a509cc982102f587cf06840f02327 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Jul 2020 16:06:24 +0000 Subject: [PATCH 091/207] Narrow scope of temporary value --- src/libstore/daemon.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index db7139374..6e0b290ed 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -375,21 +375,24 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); StringSink savedNAR; TeeSource savedNARSource(from, savedNAR); From c86fc3a9657096b74fe967f2f0bbd120e46908f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Jul 2020 15:55:04 +0000 Subject: [PATCH 092/207] Crudely make `addToStoreFromDump` take `Source` not string I just as little beyond the type as possible, so the implementation changes this enables can be reviewed separately. --- src/libstore/build.cc | 2 +- src/libstore/daemon.cc | 5 ++++- src/libstore/local-store.cc | 5 ++++- src/libstore/local-store.hh | 2 +- src/libstore/store-api.hh | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ac2e67574..62294a08c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2774,7 +2774,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6e0b290ed..69d7ef511 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -410,8 +410,11 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); + StringSource dumpSource { + method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s + }; auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, + dumpSource, baseName, method, hashAlgo); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 26b226fe8..603f36352 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1033,9 +1033,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStoreFromDump(Source & dumpSource, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { + // FIXME: See if we can use the original source to reduce memory usage. + auto dump = dumpSource.drain(); + Hash h = hashString(hashAlgo, dump); auto dstPath = makeFixedOutputPath(method, h, name); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c0e5d0286..355c2814f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -153,7 +153,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a4be0411e..d1cb2035f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -460,7 +460,7 @@ public: std::optional expectedCAHash = {}); // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); From 9de96ef7d409fedea092045c4dbae7177f88962a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Jul 2020 19:03:39 +0000 Subject: [PATCH 093/207] Dedup `LocalStore::addToStore*` The downsides is that the coroutine has byte-by-byte loop transfer. Will fix that next. --- src/libstore/local-store.cc | 83 ++++++++++--------------------------- src/libstore/local-store.hh | 4 ++ 2 files changed, 25 insertions(+), 62 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 603f36352..925ac25bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1033,65 +1033,16 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & dumpSource, const string & name, +StorePath LocalStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - // FIXME: See if we can use the original source to reduce memory usage. - auto dump = dumpSource.drain(); - - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(method, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); + return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink) { + while (1) { + uint8_t buf[1]; + auto n = dump.read(buf, 1); + sink(buf, n); } - - outputLock.setDeletion(true); - } - - return dstPath; + }); } @@ -1100,6 +1051,19 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); + return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); +} + + +StorePath LocalStore::addToStoreCommon( + const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, + std::function demux) +{ /* For computing the NAR hash. */ auto sha256Sink = std::make_unique(htSHA256); @@ -1120,7 +1084,6 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, std::string nar; auto source = sinkToSource([&](Sink & sink) { - LambdaSink sink2([&](const unsigned char * buf, size_t len) { (*sha256Sink)(buf, len); if (hashSink) (*hashSink)(buf, len); @@ -1138,11 +1101,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (!inMemory) sink(buf, len); }); - - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink2, filter); - else - readFile(srcPath, sink2); + demux(sink2); }); std::unique_ptr delTempDir; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 355c2814f..215731f87 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -290,6 +290,10 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); + StorePath addToStoreCommon( + const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, + std::function demux); + Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; From 592851fb67cd15807109d6f65fb81f6af89af966 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Jul 2020 23:40:49 +0000 Subject: [PATCH 094/207] LocalStore::addToStoreFromDump copy in chunks Rather than copying byte-by-byte, we let the coroutine know how much data we would like it to send back to us. --- src/libstore/local-store.cc | 16 +++++++++------- src/libstore/local-store.hh | 2 +- src/libutil/serialise.cc | 33 ++++++++++++++++++++------------- src/libutil/serialise.hh | 11 ++++++++++- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 925ac25bf..dac7a50c4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1036,11 +1036,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink) { + return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & wanted) { while (1) { - uint8_t buf[1]; - auto n = dump.read(buf, 1); + constexpr size_t bufSize = 1024; + uint8_t buf[bufSize]; + auto n = dump.read(buf, std::min(wanted, bufSize)); sink(buf, n); + // when control is yielded back to us wanted will be updated. } }); } @@ -1051,7 +1053,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink) { + return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & _) { if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else @@ -1062,7 +1064,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, StorePath LocalStore::addToStoreCommon( const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, - std::function demux) + std::function demux) { /* For computing the NAR hash. */ auto sha256Sink = std::make_unique(htSHA256); @@ -1083,7 +1085,7 @@ StorePath LocalStore::addToStoreCommon( bool inMemory = true; std::string nar; - auto source = sinkToSource([&](Sink & sink) { + auto source = sinkToSource([&](Sink & sink, size_t & wanted) { LambdaSink sink2([&](const unsigned char * buf, size_t len) { (*sha256Sink)(buf, len); if (hashSink) (*hashSink)(buf, len); @@ -1101,7 +1103,7 @@ StorePath LocalStore::addToStoreCommon( if (!inMemory) sink(buf, len); }); - demux(sink2); + demux(sink2, wanted); }); std::unique_ptr delTempDir; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 215731f87..ae23004c4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -292,7 +292,7 @@ private: StorePath addToStoreCommon( const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, - std::function demux); + std::function demux); Path getRealStoreDir() override { return realStoreDir; } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188f..141e9e976 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -165,35 +165,43 @@ size_t StringSource::read(unsigned char * data, size_t len) #endif std::unique_ptr sinkToSource( - std::function fun, + std::function fun, std::function eof) { struct SinkToSource : Source { - typedef boost::coroutines2::coroutine coro_t; + typedef boost::coroutines2::coroutine> coro_t; - std::function fun; + std::function fun; std::function eof; std::optional coro; bool started = false; - SinkToSource(std::function fun, std::function eof) + /* It would be nicer to have the co-routines have both args and a + return value, but unfortunately that was removed from Boost's + implementation for some reason, so we use some extra state instead. + */ + size_t wanted = 0; + + SinkToSource(std::function fun, std::function eof) : fun(fun), eof(eof) { } - std::string cur; + std::basic_string cur; size_t pos = 0; size_t read(unsigned char * data, size_t len) override { - if (!coro) + wanted = len < cur.size() ? 0 : len - cur.size(); + if (!coro) { coro = coro_t::pull_type([&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); + LambdaSink sink([&](const uint8_t * data, size_t len) { + if (len) yield(std::basic_string { data, len }); }); - fun(sink); + fun(sink, wanted); }); + } if (!*coro) { eof(); abort(); } @@ -203,11 +211,10 @@ std::unique_ptr sinkToSource( pos = 0; } - auto n = std::min(cur.size() - pos, len); - memcpy(data, (unsigned char *) cur.data() + pos, n); - pos += n; + auto numCopied = cur.copy(data, len, pos); + pos += numCopied; - return n; + return numCopied; } }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8386a4991..6cb9d1bf5 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -260,11 +260,20 @@ struct LambdaSource : Source /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ std::unique_ptr sinkToSource( - std::function fun, + std::function fun, std::function eof = []() { throw EndOfFile("coroutine has finished"); }); +static inline std::unique_ptr sinkToSource( + std::function fun, + std::function eof = []() { + throw EndOfFile("coroutine has finished"); + }) +{ + return sinkToSource([fun](Sink & s, size_t & _) { fun(s); }, eof); +} + void writePadding(size_t len, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); From 8173e7bfefc6a5771b2c9ec48bd6edd3b161dd90 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Jul 2020 21:11:08 +0000 Subject: [PATCH 095/207] Fix localhost::addToStore(...Path...) We were calculating the nar hash wrong when the file ingestion method was flat. I don't think there's anything we can do in that case but dump the file again, so that's what I do. As an optomization, we again could reuse the original dump for just the recursive and non-sha256 case, but I rather do that after this fix, and after my other PRs which deduplicate this code. --- src/libstore/local-store.cc | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 26b226fe8..5827dfc58 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1097,15 +1097,8 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique(htSHA256); - - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr hashSink = - method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 - ? nullptr - : std::make_unique(hashAlgo); + /* For computing the store path. */ + auto hashSink = std::make_unique(hashAlgo); /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1114,13 +1107,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, temporary path. Otherwise, we move it to the destination store path. */ bool inMemory = true; - std::string nar; + std::string nar; // TODO rename from "nar" to "dump" auto source = sinkToSource([&](Sink & sink) { LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); + (*hashSink)(buf, len); if (inMemory) { if (nar.size() + len > settings.narBufferSize) { @@ -1165,9 +1157,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, /* The NAR fits in memory, so we didn't do restorePath(). */ } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); auto dstPath = makeFixedOutputPath(method, hash, name); @@ -1201,13 +1191,22 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); ValidPathInfo info(dstPath); - info.narHash = sha256.first; - info.narSize = sha256.second; + info.narHash = narHash.first; + info.narSize = narHash.second; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } From 650c2c655810c375296b52997e2f85298c7c566a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Jul 2020 21:28:50 +0000 Subject: [PATCH 096/207] Rename variable `nar` -> `dump` according to TODO --- src/libstore/local-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5827dfc58..cd92f138c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1107,7 +1107,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, temporary path. Otherwise, we move it to the destination store path. */ bool inMemory = true; - std::string nar; // TODO rename from "nar" to "dump" + std::string dump; auto source = sinkToSource([&](Sink & sink) { @@ -1115,13 +1115,13 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, (*hashSink)(buf, len); if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { + if (dump.size() + len > settings.narBufferSize) { inMemory = false; sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); + sink((const unsigned char *) dump.data(), dump.size()); + dump.clear(); } else { - nar.append((const char *) buf, len); + dump.append((const char *) buf, len); } } @@ -1180,7 +1180,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (inMemory) { /* Restore from the NAR in memory. */ - StringSource source(nar); + StringSource source(dump); if (method == FileIngestionMethod::Recursive) restorePath(realPath, source); else From d087cf48552ee82e2bc78bb6c99854bab350ee00 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 15 Jul 2020 21:10:33 +0000 Subject: [PATCH 097/207] Revert "Revert "LocalStore::addToStore(srcPath): Handle the flat case"" This reverts commit cff2157185912025c24a1b9dc99056161634176c. --- src/libstore/local-store.cc | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b3f4b3f7d..26b226fe8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1097,16 +1097,13 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - /* For computing the NAR hash. */ auto sha256Sink = std::make_unique(htSHA256); /* For computing the store path. In recursive SHA-256 mode, this is the same as the NAR hash, so no need to do it again. */ std::unique_ptr hashSink = - hashAlgo == htSHA256 + method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 ? nullptr : std::make_unique(hashAlgo); @@ -1139,7 +1136,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (!inMemory) sink(buf, len); }); - dumpPath(srcPath, sink2, filter); + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink2, filter); + else + readFile(srcPath, sink2); }); std::unique_ptr delTempDir; @@ -1155,7 +1155,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, *source); + else + writeFile(tempPath, *source); } catch (EndOfFile &) { if (!inMemory) throw; @@ -1188,7 +1191,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (inMemory) { /* Restore from the NAR in memory. */ StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, source); + else + writeFile(realPath, source); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) From bc109648c41f8021707b55b815e68a890a09f2f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 15 Jul 2020 23:14:30 +0000 Subject: [PATCH 098/207] Get rid of `LocalStore::addToStoreCommon` I got it to just become `LocalStore::addToStoreFromDump`, cleanly taking a store and then doing nothing too fancy with it. `LocalStore::addToStore(...Path...)` is now just a simple wrapper with a bare-bones sinkToSource of the right dump command. --- src/libstore/local-store.cc | 93 ++++++++++++++++--------------------- src/libstore/local-store.hh | 4 -- src/libutil/serialise.cc | 13 ++++++ src/libutil/serialise.hh | 15 +++++- 4 files changed, 67 insertions(+), 58 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b9fae6089..07e1679da 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1033,38 +1033,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & wanted) { - while (1) { - constexpr size_t bufSize = 1024; - uint8_t buf[bufSize]; - auto n = dump.read(buf, std::min(wanted, bufSize)); - sink(buf, n); - // when control is yielded back to us wanted will be updated. - } - }); -} - - StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); - - return addToStoreCommon(name, method, hashAlgo, repair, [&](auto & sink, size_t & _) { + auto source = sinkToSource([&](Sink & sink, size_t & wanted) { if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else readFile(srcPath, sink); }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); } -StorePath LocalStore::addToStoreCommon( - const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, - std::function demux) +StorePath LocalStore::addToStoreFromDump(Source & source, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1075,50 +1059,53 @@ StorePath LocalStore::addToStoreCommon( destination store path is already valid, we just delete the temporary path. Otherwise, we move it to the destination store path. */ - bool inMemory = true; + bool inMemory = false; + std::string dump; - auto source = sinkToSource([&](Sink & sink, size_t & wanted) { - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*hashSink)(buf, len); - - if (inMemory) { - if (dump.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) dump.data(), dump.size()); - dump.clear(); - } else { - dump.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - demux(sink2, wanted); - }); + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 1024; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + /* Start hashing as we get data */ + (*hashSink)((const uint8_t *) dump.data() + oldSize, got); + dump.resize(oldSize + got); + } std::unique_ptr delTempDir; Path tempPath; - try { - /* Wait for the source coroutine to give us some dummy - data. This is so that we don't create the temporary - directory if the NAR fits in memory. */ - readInt(*source); + if (!inMemory) { + StringSource dumpSource { dump }; + TeeSource rest { source, *hashSink }; + ChainSource bothSource { + .source1 = dumpSource, + /* Continue hashing what's left, but don't rehash what we + already did. */ + .source2 = rest, + }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, *source); + restorePath(tempPath, bothSource); else - writeFile(tempPath, *source); + writeFile(tempPath, bothSource); - } catch (EndOfFile &) { - if (!inMemory) throw; - /* The NAR fits in memory, so we didn't do restorePath(). */ + dump.clear(); } auto [hash, size] = hashSink->finish(); @@ -1143,12 +1130,12 @@ StorePath LocalStore::addToStoreCommon( autoGC(); if (inMemory) { + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - StringSource source(dump); if (method == FileIngestionMethod::Recursive) - restorePath(realPath, source); + restorePath(realPath, dumpSource); else - writeFile(realPath, source); + writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae23004c4..355c2814f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -290,10 +290,6 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); - StorePath addToStoreCommon( - const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, - std::function demux); - Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 141e9e976..4c72dc9f2 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -329,5 +329,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6cb9d1bf5..3e3735ca5 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -256,6 +256,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ @@ -271,7 +284,7 @@ static inline std::unique_ptr sinkToSource( throw EndOfFile("coroutine has finished"); }) { - return sinkToSource([fun](Sink & s, size_t & _) { fun(s); }, eof); + return sinkToSource([fun](Sink & s, size_t & _) { fun(s); }, eof); } From 5602637d9ea195784368e99a226718fc95e6b978 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 15 Jul 2020 23:19:41 +0000 Subject: [PATCH 099/207] Revert "LocalStore::addToStoreFromDump copy in chunks" This reverts commit 592851fb67cd15807109d6f65fb81f6af89af966. We don't need this extra feature anymore --- src/libstore/local-store.cc | 2 +- src/libutil/serialise.cc | 33 +++++++++++++-------------------- src/libutil/serialise.hh | 11 +---------- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 07e1679da..b2b5afadd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1037,7 +1037,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); - auto source = sinkToSource([&](Sink & sink, size_t & wanted) { + auto source = sinkToSource([&](Sink & sink) { if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 4c72dc9f2..00c945113 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -165,43 +165,35 @@ size_t StringSource::read(unsigned char * data, size_t len) #endif std::unique_ptr sinkToSource( - std::function fun, + std::function fun, std::function eof) { struct SinkToSource : Source { - typedef boost::coroutines2::coroutine> coro_t; + typedef boost::coroutines2::coroutine coro_t; - std::function fun; + std::function fun; std::function eof; std::optional coro; bool started = false; - /* It would be nicer to have the co-routines have both args and a - return value, but unfortunately that was removed from Boost's - implementation for some reason, so we use some extra state instead. - */ - size_t wanted = 0; - - SinkToSource(std::function fun, std::function eof) + SinkToSource(std::function fun, std::function eof) : fun(fun), eof(eof) { } - std::basic_string cur; + std::string cur; size_t pos = 0; size_t read(unsigned char * data, size_t len) override { - wanted = len < cur.size() ? 0 : len - cur.size(); - if (!coro) { + if (!coro) coro = coro_t::pull_type([&](coro_t::push_type & yield) { - LambdaSink sink([&](const uint8_t * data, size_t len) { - if (len) yield(std::basic_string { data, len }); + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); }); - fun(sink, wanted); + fun(sink); }); - } if (!*coro) { eof(); abort(); } @@ -211,10 +203,11 @@ std::unique_ptr sinkToSource( pos = 0; } - auto numCopied = cur.copy(data, len, pos); - pos += numCopied; + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char *) cur.data() + pos, n); + pos += n; - return numCopied; + return n; } }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3e3735ca5..aa6b42597 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -273,19 +273,10 @@ struct ChainSource : Source /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ std::unique_ptr sinkToSource( - std::function fun, - std::function eof = []() { - throw EndOfFile("coroutine has finished"); - }); - -static inline std::unique_ptr sinkToSource( std::function fun, std::function eof = []() { throw EndOfFile("coroutine has finished"); - }) -{ - return sinkToSource([fun](Sink & s, size_t & _) { fun(s); }, eof); -} + }); void writePadding(size_t len, Sink & sink); From 68dfb8c6aef7afebf0312c48bb5010653fc464b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Jul 2020 05:09:41 +0000 Subject: [PATCH 100/207] Optimize `addToStoreSlow` and remove `TeeParseSink` --- src/libstore/daemon.cc | 47 +++++++---------------------------- src/libstore/export-import.cc | 12 +++++---- src/libstore/store-api.cc | 35 ++++++++++++++++++++------ src/libutil/archive.hh | 25 ++++++++++++++++--- 4 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index db7139374..573836f7f 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource } }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - struct ClientSettings { bool keepFailed; @@ -391,9 +366,9 @@ static void performOp(TunnelLogger * logger, ref store, } HashType hashAlgo = parseHashType(s); - StringSink savedNAR; - TeeSource savedNARSource(from, savedNAR); - RetrieveRegularNARSink savedRegular; + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to @@ -407,11 +382,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, - baseName, - method, - hashAlgo); + auto path = store->addToStoreFromDump(*saved.s, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -727,15 +698,15 @@ static void performOp(TunnelLogger * logger, ref store, if (!trusted) info.ultimate = false; - std::string saved; std::unique_ptr source; if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeParseSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.saved.s); - source = std::make_unique(saved); + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique(std::move(*saved.s)); } logger->startWork(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 082d0f1d1..b963d64d7 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -60,8 +60,10 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeParseSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -77,15 +79,15 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.saved.s); - info.narSize = tee.saved.s->size(); + info.narHash = hashString(htSHA256, *saved.s); + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.saved.s }; + auto source = StringSource { *saved.s }; addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5b9f79049..5c8dddba5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -226,16 +226,37 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, std::optional expectedCAHash) { - /* FIXME: inefficient: we're reading/hashing 'tmpFile' three + /* FIXME: inefficient: we're reading/hashing 'tmpFile' two times. */ + HashSink narHashSink { htSHA256 }; + HashSink caHashSink { hashAlgo }; + RetrieveRegularNARSink fileSink { caHashSink }; - auto [narHash, narSize] = hashPath(htSHA256, srcPath); + TeeSink sinkIfNar { narHashSink, caHashSink }; - auto hash = method == FileIngestionMethod::Recursive - ? hashAlgo == htSHA256 - ? narHash - : hashPath(hashAlgo, srcPath).first - : hashFile(hashAlgo, srcPath); + /* We use the tee sink if we need to hash he nar twice */ + auto & sink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast(sinkIfNar) + : narHashSink; + + auto fileSource = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink); + }); + + TeeSource tapped { *fileSource, sink }; + + ParseSink blank; + auto & parseSink = method == FileIngestionMethod::Flat + ? fileSink + : blank; + + parseDump(parseSink, tapped); + + auto [narHash, narSize] = narHashSink.finish(); + + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? narHash + : caHashSink.finish().first; if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 302b1bb18..57780d16a 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,12 +63,29 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeParseSink : ParseSink +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink { - StringSink saved; - TeeSource source; + bool regular = true; + Sink & sink; - TeeParseSink(Source & source) : source(source, saved) { } + RetrieveRegularNARSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + sink(data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } }; void parseDump(ParseSink & sink, Source & source); From 3dcca18c30cbc09652f5ac644a9f8750f9ced0c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Jul 2020 13:39:03 +0000 Subject: [PATCH 101/207] Fix bug in TeeSource We use this to simplify `LocalStore::addToStoreFromDump`. Also, hope I fixed build error with old clang (used in Darwin CI). --- src/libstore/local-store.cc | 14 ++++---------- src/libutil/serialise.hh | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b2b5afadd..96d10d9ba 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1047,11 +1047,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, } -StorePath LocalStore::addToStoreFromDump(Source & source, const string & name, +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); + TeeSource source { source0, *hashSink }; /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1078,8 +1079,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source, const string & name, inMemory = true; break; } - /* Start hashing as we get data */ - (*hashSink)((const uint8_t *) dump.data() + oldSize, got); dump.resize(oldSize + got); } @@ -1087,14 +1086,9 @@ StorePath LocalStore::addToStoreFromDump(Source & source, const string & name, Path tempPath; if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; - TeeSource rest { source, *hashSink }; - ChainSource bothSource { - .source1 = dumpSource, - /* Continue hashing what's left, but don't rehash what we - already did. */ - .source2 = rest, - }; + ChainSource bothSource { dumpSource, source }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique(tempDir); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index aa6b42597..5d9acf887 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -189,7 +189,7 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, len); + sink(data, n); return n; } }; From bc73590151bff82b03077c34f0c5aa9f84c89e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20M=C3=BChmel?= Date: Fri, 17 Jul 2020 17:35:59 +0200 Subject: [PATCH 102/207] nix edit: call restoreSignals() before `execvp`-ing the $EDITOR Currently resizing of the terminal doesn't play nicely with nix edit when using kakoune as the editor, as it relies on the SIGWINCH signal which is trapped by nix. How this is not a problem with e.g. vim is beyond me. Virtually all other exec* calls are following a call to restoreSignals(). This commit adds this behavior to nix edit as well. --- src/nix/edit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/edit.cc b/src/nix/edit.cc index dc9775635..378a3739c 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -45,6 +45,7 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); + restoreSignals(); execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; From 1feb8981df6adf8519a1f2d31883eb12db11fcb5 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 17 Jul 2020 11:33:27 -0400 Subject: [PATCH 103/207] Revert "Don't anticipate hash algo without hash in derivation for now" This reverts commit 3804e3df9bb479fe1d399f29d16a1aabaf352c19. --- src/libstore/derivations.cc | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bff228230..487ed47e9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -387,13 +387,34 @@ bool isDerivation(const string & fileName) DerivationType BasicDerivation::type() const { - if (outputs.size() == 1 && - outputs.begin()->first == "out" && - std::holds_alternative(outputs.begin()->second.output)) - { + std::set inputAddressedOutputs, fixedCAOutputs; + for (auto & i : outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed _) { + inputAddressedOutputs.insert(i.first); + }, + [&](DerivationOutputFixed _) { + fixedCAOutputs.insert(i.first); + }, + [&](DerivationOutputFloating _) { + throw Error("Floating CA output derivations are not yet implemented"); + }, + }, i.second.output); + } + + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty()) { + throw Error("Must have at least one output"); + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty()) { + return DerivationType::Regular; + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty()) { + if (fixedCAOutputs.size() > 1) + // FIXME: Experimental feature? + throw Error("Only one fixed output is allowed for now"); + if (*fixedCAOutputs.begin() != "out") + throw Error("Single fixed output must be named \"out\""); return DerivationType::CAFixed; } else { - return DerivationType::Regular; + throw Error("Can't mix derivation output types"); } } From 205dcd140d46db94481329578b4fee8275e6c534 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 17 Jul 2020 12:43:46 -0400 Subject: [PATCH 104/207] Revert "Don't anticipate multiple CA outputs for now" This reverts commit 74b251b2f3d6414de051c8523011c0ee3c5ea154. --- src/libstore/local-store.cc | 43 ++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 19537b8e5..9117ff384 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -544,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) { - if (actual != expected) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); auto j = drv.env.find(varName); if (j == drv.env.end() || parseStorePath(j->second) != actual) throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", @@ -556,18 +553,34 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - if (derivationIsFixed(drv.type())) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - } + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + drv.type(); - else { - // Regular, non-CA derivation should always return a single hash and not - // hash per output. - Hash h = std::get<0>(hashDerivationModulo(*this, drv, true)); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); + std::optional h; + for (auto & i : drv.outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed doia) { + if (!h) { + // somewhat expensive so we do lazily + auto temp = hashDerivationModulo(*this, drv, true); + h = std::get(temp); + } + StorePath recomputed = makeOutputPath(i.first, *h, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](DerivationOutputFixed dof) { + StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](DerivationOutputFloating _) { + throw Error("Floating CA output derivations are not yet implemented"); + }, + }, i.second.output); } } From bbc633c98ca2c2f11303efafe4d58edd6d9b1018 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 17 Jul 2020 13:10:32 -0400 Subject: [PATCH 105/207] Revert "Don't anticipate CA but not fixed outputs for now" This reverts commit 3a9e4c32624b36b70cf8d553fd76a85ee97773ab. --- src/libstore/build.cc | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 28f3ead75..b59bfa8bc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3721,9 +3721,22 @@ void DerivationGoal::registerOutputs() hash). */ std::optional ca; - if (derivationIsFixed(derivationType)) { - - FixedOutputHash outputHash = std::get(i.second.output).hash; + if (! std::holds_alternative(i.second.output)) { + DerivationOutputFloating outputHash; + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + throw Error("No."); + }, + [&](DerivationOutputFixed dof) { + outputHash = DerivationOutputFloating { + .method = dof.hash.method, + .hashType = *dof.hash.hash.type, + }; + }, + [&](DerivationOutputFloating dof) { + outputHash = dof; + }, + }, i.second.output); if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ @@ -3737,12 +3750,17 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = outputHash.method == FileIngestionMethod::Recursive - ? hashPath(*outputHash.hash.type, actualPath).first - : hashFile(*outputHash.hash.type, actualPath); + ? hashPath(outputHash.hashType, actualPath).first + : hashFile(outputHash.hashType, actualPath); auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); - if (outputHash.hash != h2) { + // true if either floating CA, or incorrect fixed hash. + bool needsMove = true; + + if (auto p = std::get_if(& i.second.output)) { + Hash & h = p->hash.hash; + if (h != h2) { /* Throw an error after registering the path as valid. */ @@ -3750,9 +3768,15 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - outputHash.hash.to_string(SRI, true), + h.to_string(SRI, true), h2.to_string(SRI, true))); + } else { + // matched the fixed hash, so no move needed. + needsMove = false; + } + } + if (needsMove) { Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) From 6756cecfcf266801219b1e2da7e79f14695ccecf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Jul 2020 19:55:41 +0000 Subject: [PATCH 106/207] Add `DerivationType::CAFloating` --- src/libstore/derivations.cc | 28 +++++++++++++++++++++------- src/libstore/derivations.hh | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 487ed47e9..2a95c7e69 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -28,6 +28,7 @@ bool derivationIsCA(DerivationType dt) { switch (dt) { case DerivationType::Regular: return false; case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return true; }; // Since enums can have non-variant values, but making a `default:` would // disable exhaustiveness warnings. @@ -38,6 +39,7 @@ bool derivationIsFixed(DerivationType dt) { switch (dt) { case DerivationType::Regular: return false; case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; }; abort(); } @@ -46,6 +48,7 @@ bool derivationIsImpure(DerivationType dt) { switch (dt) { case DerivationType::Regular: return false; case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; }; abort(); } @@ -387,7 +390,8 @@ bool isDerivation(const string & fileName) DerivationType BasicDerivation::type() const { - std::set inputAddressedOutputs, fixedCAOutputs; + std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { [&](DerivationOutputInputAddressed _) { @@ -396,23 +400,31 @@ DerivationType BasicDerivation::type() const [&](DerivationOutputFixed _) { fixedCAOutputs.insert(i.first); }, - [&](DerivationOutputFloating _) { - throw Error("Floating CA output derivations are not yet implemented"); + [&](DerivationOutputFloating dof) { + floatingCAOutputs.insert(i.first); + if (!floatingHashType) { + floatingHashType = dof.hashType; + } else { + if (*floatingHashType != dof.hashType) + throw Error("All floating outputs must use the same hash type"); + } }, }, i.second.output); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty()) { + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty()) { + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { return DerivationType::Regular; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? throw Error("Only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") throw Error("Single fixed output must be named \"out\""); return DerivationType::CAFixed; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + return DerivationType::CAFloating; } else { throw Error("Can't mix derivation output types"); } @@ -464,6 +476,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m { /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { + case DerivationType::CAFloating: + throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -476,7 +490,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } - default: + case DerivationType::Regular: break; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 45217b3d5..b8e2494a3 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -61,6 +61,7 @@ typedef std::map StringPairs; enum struct DerivationType : uint8_t { Regular, CAFixed, + CAFloating, }; /* Do the outputs of the derivation have paths calculated from their content, From 5ae747b43498076efe9b60d1a871f9956dd3fc84 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Jul 2020 21:50:09 +0000 Subject: [PATCH 107/207] Remove stray added file --- local-store-temp.cc | 1787 ------------------------------------------- 1 file changed, 1787 deletions(-) delete mode 100644 local-store-temp.cc diff --git a/local-store-temp.cc b/local-store-temp.cc deleted file mode 100644 index be6e081cb..000000000 --- a/local-store-temp.cc +++ /dev/null @@ -1,1787 +0,0 @@ -#include "local-store.hh" -#include "globals.hh" -#include "git.hh" -#include "archive.hh" -#include "pathlocks.hh" -#include "worker-protocol.hh" -#include "derivations.hh" -#include "nar-info.hh" -#include "references.hh" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if __linux__ -#include -#include -#include -#include -#include -#endif - -#ifdef __CYGWIN__ -#include -#endif - -#include - - -namespace nix { - - -LocalStore::LocalStore(const Params & params) - : Store(params) - , LocalFSStore(params) - , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"} - , realStoreDir(realStoreDir_) - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") - , reservedPath(dbDir + "/reserved") - , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") - , tempRootsDir(stateDir + "/temproots") - , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) - , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) -{ - auto state(_state.lock()); - - /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - makeStoreWritable(); - createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; - createDirs(profilesDir); - createDirs(tempRootsDir); - createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; - if (!pathExists(gcRootsDir)) { - createDirs(gcRootsDir); - createSymlink(profilesDir, gcRootsDir + "/profiles"); - } - - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); - } - - createUser(getUserName(), getuid()); - - /* Optionally, create directories and set permissions for a - multi-user install. */ - if (getuid() == 0 && settings.buildUsersGroup != "") { - mode_t perm = 01775; - - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - logError({ - .name = "'build-users-group' not found", - .hint = hintfmt( - "warning: the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup) - }); - else { - struct stat st; - if (stat(realStoreDir.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); - - if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); - } - } - } - - /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { - Path path = realStoreDir; - struct stat st; - while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - if (S_ISLNK(st.st_mode)) - throw Error( - "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories", - path); - path = dirOf(path); - } - } - - /* We can't open a SQLite database if the disk is full. Since - this prevents the garbage collector from running when it's most - needed, we reserve some dummy space that we can free just - before doing a garbage collection. */ - try { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); - int res = -1; -#if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd.get(), 0, settings.reservedSize); -#endif - if (res == -1) { - writeFull(fd.get(), string(settings.reservedSize, 'X')); - [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); - } - } - } catch (SysError & e) { /* don't care about errors */ - } - - /* Acquire the big fat lock in shared mode to make sure that no - schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); - - if (!lockFile(globalLock.get(), ltRead, false)) { - printInfo("waiting for the big Nix store lock..."); - lockFile(globalLock.get(), ltRead, true); - } - - /* Check the current database schema and if necessary do an - upgrade. */ - int curSchema = getSchema(); - if (curSchema > nixSchemaVersion) - throw Error("current Nix store schema is version %1%, but I only support %2%", - curSchema, nixSchemaVersion); - - else if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; - openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - } - - else if (curSchema < nixSchemaVersion) { - if (curSchema < 5) - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - - if (curSchema < 6) - throw Error( - "Your Nix store has a database in flat file format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 1.11 first."); - - if (!lockFile(globalLock.get(), ltWrite, false)) { - printInfo("waiting for exclusive access to the Nix store..."); - lockFile(globalLock.get(), ltWrite, true); - } - - /* Get the schema version again, because another process may - have performed the upgrade already. */ - curSchema = getSchema(); - - if (curSchema < 7) { upgradeStore7(); } - - openDB(*state, false); - - if (curSchema < 8) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ultimate integer"); - state->db.exec("alter table ValidPaths add column sigs text"); - txn.commit(); - } - - if (curSchema < 9) { - SQLiteTxn txn(state->db); - state->db.exec("drop table FailedPaths"); - txn.commit(); - } - - if (curSchema < 10) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ca text"); - txn.commit(); - } - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock.get(), ltRead, true); - } - - else openDB(*state, false); - - /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create(state->db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create(state->db, - "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); - state->stmtAddReference.create(state->db, - "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create(state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, - "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - state->stmtQueryReferrers.create(state->db, - "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, - "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create(state->db, - "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - state->stmtQueryValidDerivers.create(state->db, - "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create(state->db, - "select id, path from DerivationOutputs where drv = ?;"); - // Use "path >= ?" with limit 1 rather than "path like '?%'" to - // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create(state->db, - "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); -} - - -LocalStore::~LocalStore() -{ - std::shared_future future; - - { - auto state(_state.lock()); - if (state->gcRunning) - future = state->gcFuture; - } - - if (future.valid()) { - printInfo("waiting for auto-GC to finish on exit..."); - future.get(); - } - - try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; - unlink(fnTempRoots.c_str()); - } - } catch (...) { - ignoreException(); - } -} - - -std::string LocalStore::getUri() -{ - return "local"; -} - - -int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error("'%1%' is corrupt", schemaPath); - } - return curSchema; -} - - -void LocalStore::openDB(State & state, bool create) -{ - if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError("Nix database directory '%1%' is not writable", dbDir); - - /* Open the Nix database. */ - string dbPath = dbDir + "/db.sqlite"; - auto & db(state.db); - state.db = SQLite(dbPath, create); - -#ifdef __CYGWIN__ - /* The cygwin version of sqlite3 has a patch which calls - SetDllDirectory("/usr/bin") on init. It was intended to fix extension - loading, which we don't use, and the effect of SetDllDirectory is - inherited by child processes, and causes libraries to be loaded from - /usr/bin instead of $PATH. This breaks quite a few things (e.g. - checkPhase on openssh), so we set it back to default behaviour. */ - SetDllDirectoryW(L""); -#endif - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - string syncMode = settings.fsyncMetadata ? "normal" : "off"; - db.exec("pragma synchronous = " + syncMode); - - /* Set the SQLite journal mode. WAL mode is fastest, so it's the - default. */ - string mode = settings.useSQLiteWAL ? "wal" : "truncate"; - string prevMode; - { - SQLiteStmt stmt; - stmt.create(db, "pragma main.journal_mode;"); - if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); - prevMode = string((const char *) sqlite3_column_text(stmt, 0)); - } - if (prevMode != mode && - sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); - - /* Increase the auto-checkpoint interval to 40000 pages. This - seems enough to ensure that instantiating the NixOS system - derivation is done in a single fsync(). */ - if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); - - /* Initialise the database schema, if necessary. */ - if (create) { - static const char schema[] = -#include "schema.sql.gen.hh" - ; - db.exec(schema); - } -} - - -/* To improve purity, users may want to make the Nix store a read-only - bind mount. So make the Nix store writable for this process. */ -void LocalStore::makeStoreWritable() -{ -#if __linux__ - if (getuid() != 0) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); - } -#endif -} - - -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - canonicaliseTimestampAndPermissions(path, st); -} - - -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); - } -} - - -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, fromUid, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, uid_t fromUid) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); -} - - -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - - auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) - { - if (actual != expected) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); - } - - else { - Hash h = hashDerivationModulo(*this, drv, true); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); - } -} - - -uint64_t LocalStore::addValidPath(State & state, - const ValidPathInfo & info, bool checkOutputs) -{ - if (info.ca.has_value() && !info.isContentAddressed(*this)) - throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", - printStorePath(info.path)); - - state.stmtRegisterValidPath.use() - (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) - (info.registrationTime == 0 ? time(0) : info.registrationTime) - (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) - (info.narSize, info.narSize != 0) - (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) - .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); - - /* If this is a derivation, then store the derivation outputs in - the database. This is useful for the garbage collector: it can - efficiently query whether a path is an output of some - derivation. */ - if (info.path.isDerivation()) { - auto drv = readDerivation(info.path); - - /* Verify that the output paths in the derivation are correct - (i.e., follow the scheme for computing output paths from - derivations). Note that if this throws an error, then the - DB transaction is rolled back, so the path validity - registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); - - for (auto & i : drv.outputs) { - state.stmtAddDerivationOutput.use() - (id) - (i.first) - (printStorePath(i.second.path)) - .exec(); - } - } - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(std::string(info.path.hashPart()), - PathInfoCacheValue{ .value = std::make_shared(info) }); - } - - return id; -} - - -void LocalStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept -{ - try { - auto info = std::make_shared(path); - - callback(retrySQLite>([&]() { - auto state(_state.lock()); - - /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); - - if (!useQueryPathInfo.next()) - return std::shared_ptr(); - - info->id = useQueryPathInfo.getInt(0); - - try { - info->narHash = Hash(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); - } - - info->registrationTime = useQueryPathInfo.getInt(2); - - auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); - - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); - - info->ultimate = useQueryPathInfo.getInt(5) == 1; - - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); - - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); - - /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - - return info; - })); - - } catch (...) { callback.rethrow(); } -} - - -/* Update path info in the database. */ -void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) -{ - state.stmtUpdatePathInfo.use() - (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) - (info.ultimate ? 1 : 0, info.ultimate) - (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) - (printStorePath(info.path)) - .exec(); -} - - -uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) -{ - auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); - if (!use.next()) - throw Error("path '%s' is not valid", printStorePath(path)); - return use.getInt(0); -} - - -bool LocalStore::isValidPath_(State & state, const StorePath & path) -{ - return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); -} - - -bool LocalStore::isValidPathUncached(const StorePath & path) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - return isValidPath_(*state, path); - }); -} - - -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) -{ - StorePathSet res; - for (auto & i : paths) - if (isValidPath(i)) res.insert(i); - return res; -} - - -StorePathSet LocalStore::queryAllValidPaths() -{ - return retrySQLite([&]() { - auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); - StorePathSet res; - while (use.next()) res.insert(parseStorePath(use.getStr(0))); - return res; - }); -} - - -void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) -{ - auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); - - while (useQueryReferrers.next()) - referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); -} - - -void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - queryReferrers(*state, path, referrers); - }); -} - - -StorePathSet LocalStore::queryValidDerivers(const StorePath & path) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); - - StorePathSet derivers; - while (useQueryValidDerivers.next()) - derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1))); - - return derivers; - }); -} - - -OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - OutputPathMap outputs; - while (useQueryDerivationOutputs.next()) - outputs.emplace( - useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1)) - ); - - return outputs; - }); -} - - -std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) -{ - if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); - - Path prefix = storeDir + "/" + hashPart; - - return retrySQLite>([&]() -> std::optional { - auto state(_state.lock()); - - auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - - if (!useQueryPathFromHashPart.next()) return {}; - - const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); - if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) - return parseStorePath(s); - return {}; - }); -} - - -StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) -{ - if (!settings.useSubstitutes) return StorePathSet(); - - StorePathSet remaining; - for (auto & i : paths) - remaining.insert(i); - - StorePathSet res; - - for (auto & sub : getDefaultSubstituters()) { - if (remaining.empty()) break; - if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; - - auto valid = sub->queryValidPaths(remaining); - - StorePathSet remaining2; - for (auto & path : remaining) - if (valid.count(path)) - res.insert(path); - else - remaining2.insert(path); - - std::swap(remaining, remaining2); - } - - return res; -} - - -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; - for (auto & path : paths) { - if (infos.count(path)) continue; - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); - try { - auto info = sub->queryPathInfo(path); - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - -void LocalStore::registerValidPath(const ValidPathInfo & info) -{ - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); -} - - -void LocalStore::registerValidPaths(const ValidPathInfos & infos) -{ - /* SQLite will fsync by default, but the new valid paths may not - be fsync-ed. So some may want to fsync them before registering - the validity, at the expense of some speed of the path - registering operation. */ - if (settings.syncBeforeRegistering) sync(); - - return retrySQLite([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - StorePathSet paths; - - for (auto & i : infos) { - assert(i.narHash.type == htSHA256); - if (isValidPath_(*state, i.path)) - updatePathInfo(*state, i); - else - addValidPath(*state, i, false); - paths.insert(i.path); - } - - for (auto & i : infos) { - auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); - } - - /* Check that the derivation outputs are correct. We can't do - this in addValidPath() above, because the references might - not be valid yet. */ - for (auto & i : infos) - if (i.path.isDerivation()) { - // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, readDerivation(i.path)); - } - - /* Do a topological sort of the paths. This will throw an - error if a cycle is detected and roll back the - transaction. Cycles can only occur when a derivation - has multiple outputs. */ - topoSortPaths(paths); - - txn.commit(); - }); -} - - -/* Invalidate a path. The caller is responsible for checking that - there are no referrers. */ -void LocalStore::invalidatePath(State & state, const StorePath & path) -{ - debug("invalidating path '%s'", printStorePath(path)); - - state.stmtInvalidatePath.use()(printStorePath(path)).exec(); - - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.hashPart())); - } -} - - -const PublicKeys & LocalStore::getPublicKeys() -{ - auto state(_state.lock()); - if (!state->publicKeys) - state->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *state->publicKeys; -} - - -void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) -{ - if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); - - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) - throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); - - addTempRoot(info.path); - - if (repair || !isValidPath(info.path)) { - - PathLocks outputLock; - - auto realPath = Store::toRealPath(info.path); - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - if (!locksHeld.count(printStorePath(info.path))) - outputLock.lockPaths({realPath}); - - if (repair || !isValidPath(info.path)) { - - deletePath(realPath); - - // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !refersToSelf)) - settings.requireExperimentalFeature("ca-references"); - - /* While restoring the path from the NAR, compute the hash - of the NAR. */ - std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) - hashSink = std::make_unique(htSHA256); - else - hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); - - LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { - size_t n = source.read(data, len); - (*hashSink)(data, n); - return n; - }); - - restorePath(realPath, wrapperSource); - - auto hashResult = hashSink->finish(); - - if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); - - if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narSize, hashResult.second); - - autoGC(); - - canonicalisePathMetaData(realPath, -1); - - optimisePath(realPath); // FIXME: combine with hashPath() - - registerValidPath(info); - } - - outputLock.setDeletion(true); - } -} - - -<<<<<<< HEAD -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) - throw Error("git ingestion must use sha1 hash"); - - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(method, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - switch (method) { - case FileIngestionMethod::Flat: - writeFile(realPath, dump); - break; - case FileIngestionMethod::Recursive: { - StringSource source(dump); - restorePath(realPath, source); - break; - } - case FileIngestionMethod::Git: { - StringSource source(dump); - restoreGit(realPath, source, realStoreDir, storeDir); - break; - } - } - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -||||||| merged common ancestors -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(method, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -======= ->>>>>>> 3dcca18c30cbc09652f5ac644a9f8750f9ced0c9 -StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); - }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); -} - - -StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - /* For computing the store path. */ - auto hashSink = std::make_unique(hashAlgo); - TeeSource source { source0, *hashSink }; - - /* Read the source path into memory, but only if it's up to - narBufferSize bytes. If it's larger, write it to a temporary - location in the Nix store. If the subsequently computed - destination store path is already valid, we just delete the - temporary path. Otherwise, we move it to the destination store - path. */ -<<<<<<< HEAD - bool inMemory = true; - std::string nar; // TODO rename from "nar" to "dump" - - auto source = sinkToSource([&](Sink & sink) { - - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - switch (method) { - case FileIngestionMethod::Recursive: { - dumpPath(srcPath, sink2, filter); - break; - } - case FileIngestionMethod::Git: { - // recursively add to store if path is a directory - struct stat st; - if (lstat(srcPath.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", srcPath); - if (S_ISDIR(st.st_mode)) - for (auto & i : readDirectory(srcPath)) - addToStore("git", srcPath + "/" + i.name, method, hashAlgo, filter, repair); - - dumpGit(hashAlgo, srcPath, sink2, filter); - break; - } - case FileIngestionMethod::Flat: { - readFile(srcPath, sink2); - break; - } - } - }); -||||||| merged common ancestors - bool inMemory = true; - std::string nar; // TODO rename from "nar" to "dump" - - auto source = sinkToSource([&](Sink & sink) { - - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink2, filter); - else - readFile(srcPath, sink2); - }); -======= - bool inMemory = false; - - std::string dump; - - /* Fill out buffer, and decide whether we are working strictly in - memory based on whether we break out because the buffer is full - or the original source is empty */ - while (dump.size() < settings.narBufferSize) { - auto oldSize = dump.size(); - constexpr size_t chunkSize = 1024; - auto want = std::min(chunkSize, settings.narBufferSize - oldSize); - dump.resize(oldSize + want); - auto got = 0; - try { - got = source.read((uint8_t *) dump.data() + oldSize, want); - } catch (EndOfFile &) { - inMemory = true; - break; - } - dump.resize(oldSize + got); - } ->>>>>>> 3dcca18c30cbc09652f5ac644a9f8750f9ced0c9 - - std::unique_ptr delTempDir; - Path tempPath; - - if (!inMemory) { - /* Drain what we pulled so far, and then keep on pulling */ - StringSource dumpSource { dump }; - ChainSource bothSource { dumpSource, source }; - - auto tempDir = createTempDir(realStoreDir, "add"); - delTempDir = std::make_unique(tempDir); - tempPath = tempDir + "/x"; - -<<<<<<< HEAD - switch (method) { - case FileIngestionMethod::Flat: - writeFile(tempPath, *source); - break; - case FileIngestionMethod::Recursive: - restorePath(tempPath, *source); - break; - case FileIngestionMethod::Git: - restoreGit(tempPath, *source, realStoreDir, storeDir); - break; - } -||||||| merged common ancestors - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, *source); - else - writeFile(tempPath, *source); -======= - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); - else - writeFile(tempPath, bothSource); ->>>>>>> 3dcca18c30cbc09652f5ac644a9f8750f9ced0c9 - - dump.clear(); - } - - auto [hash, size] = hashSink->finish(); - - auto dstPath = makeFixedOutputPath(method, hash, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (inMemory) { - StringSource dumpSource { dump }; - /* Restore from the NAR in memory. */ -<<<<<<< HEAD - StringSource source(nar); - switch (method) { - case FileIngestionMethod::Flat: - writeFile(realPath, source); - break; - case FileIngestionMethod::Recursive: { - restorePath(realPath, source); - break; - } - case FileIngestionMethod::Git: { - restoreGit(realPath, source, realStoreDir, storeDir); - break; - } - } -||||||| merged common ancestors - StringSource source(nar); - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, source); - else - writeFile(realPath, source); -======= - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); - else - writeFile(realPath, dumpSource); ->>>>>>> 3dcca18c30cbc09652f5ac644a9f8750f9ced0c9 - } else { - /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); - } - - /* For computing the nar hash. In recursive SHA-256 mode, this - is the same as the store hash, so no need to do it again. */ - auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; - dumpPath(realPath, narSink); - narHash = narSink.finish(); - } - - canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath - - optimisePath(realPath); - - ValidPathInfo info(dstPath); - info.narHash = narHash.first; - info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -StorePath LocalStore::addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) -{ - auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - writeFile(realPath, s); - - canonicalisePathMetaData(realPath, -1); - - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(htSHA256, *sink.s); - - optimisePath(realPath); - - ValidPathInfo info(dstPath); - info.narHash = narHash; - info.narSize = sink.s->size(); - info.references = references; - info.ca = TextHash { .hash = hash }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -/* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() -{ - Path tmpDir; - do { - /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(parseStorePath(tmpDir)); - } while (!pathExists(tmpDir)); - return tmpDir; -} - - -void LocalStore::invalidatePathChecked(const StorePath & path) -{ - retrySQLite([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - - if (isValidPath_(*state, path)) { - StorePathSet referrers; queryReferrers(*state, path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) - throw PathInUse("cannot delete path '%s' because it is in use by %s", - printStorePath(path), showPaths(referrers)); - invalidatePath(*state, path); - } - - txn.commit(); - }); -} - - -bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) -{ - printInfo(format("reading the Nix store...")); - - bool errors = false; - - /* Acquire the global GC lock to get a consistent snapshot of - existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); - - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - - StorePathSet validPaths; - PathSet done; - - fdGCLock = -1; - - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); - - /* Optionally, check the content hashes (slow). */ - if (checkContents) { - - printInfo("checking link hashes..."); - - for (auto & link : readDirectory(linksDir)) { - printMsg(lvlTalkative, "checking contents of '%s'", link.name); - Path linkPath = linksDir + "/" + link.name; - string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); - if (hash != link.name) { - logError({ - .name = "Invalid hash", - .hint = hintfmt( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash) - }); - if (repair) { - if (unlink(linkPath.c_str()) == 0) - printInfo("removed link '%s'", linkPath); - else - throw SysError("removing corrupt link '%s'", linkPath); - } else { - errors = true; - } - } - } - - printInfo("checking store hashes..."); - - Hash nullHash(htSHA256); - - for (auto & i : validPaths) { - try { - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(i))); - - /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - - std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(*info->narHash.type); - else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); - - dumpPath(Store::toRealPath(i), *hashSink); - auto current = hashSink->finish(); - - if (info->narHash != nullHash && info->narHash != current.first) { - logError({ - .name = "Invalid hash - path modified", - .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) - }); - if (repair) repairPath(i); else errors = true; - } else { - - bool update = false; - - /* Fill in missing hashes. */ - if (info->narHash == nullHash) { - printInfo("fixing missing hash on '%s'", printStorePath(i)); - info->narHash = current.first; - update = true; - } - - /* Fill in missing narSize fields (from old stores). */ - if (info->narSize == 0) { - printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); - info->narSize = current.second; - update = true; - } - - if (update) { - auto state(_state.lock()); - updatePathInfo(*state, *info); - } - - } - - } catch (Error & e) { - /* It's possible that the path got GC'ed, so ignore - errors on invalid paths. */ - if (isValidPath(i)) - logError(e.info()); - else - warn(e.msg()); - errors = true; - } - } - } - - return errors; -} - - -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) -{ - checkInterrupt(); - - if (!done.insert(pathS).second) return; - - if (!isStorePath(pathS)) { - logError({ - .name = "Nix path not found", - .hint = hintfmt("path '%s' is not in the Nix store", pathS) - }); - return; - } - - auto path = parseStorePath(pathS); - - if (!store.count(std::string(path.to_string()))) { - /* Check any referrers first. If we can invalidate them - first, then we can invalidate this path as well. */ - bool canInvalidate = true; - StorePathSet referrers; queryReferrers(path, referrers); - for (auto & i : referrers) - if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); - if (validPaths.count(i)) - canInvalidate = false; - } - - if (canInvalidate) { - printInfo("path '%s' disappeared, removing from database...", pathS); - auto state(_state.lock()); - invalidatePath(*state, path); - } else { - logError({ - .name = "Missing path with referrers", - .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) - }); - if (repair) - try { - repairPath(path); - } catch (Error & e) { - logWarning(e.info()); - errors = true; - } - else errors = true; - } - - return; - } - - validPaths.insert(std::move(path)); -} - - -unsigned int LocalStore::getProtocol() -{ - return PROTOCOL_VERSION; -} - - -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) - -static void makeMutable(const Path & path) -{ - checkInterrupt(); - - struct stat st = lstat(path); - - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; - - if (S_ISDIR(st.st_mode)) { - for (auto & i : readDirectory(path)) - makeMutable(path + "/" + i.name); - } - - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd == -1) { - if (errno == ELOOP) return; // it's a symlink - throw SysError("opening file '%1%'", path); - } - - unsigned int flags = 0, old; - - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) return; - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; -} - -/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() -{ - if (getuid() != 0) return; - printInfo("removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(realStoreDir); -} - -#else - -void LocalStore::upgradeStore7() -{ -} - -#endif - - -void LocalStore::vacuumDB() -{ - auto state(_state.lock()); - state->db.exec("vacuum"); -} - - -void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) -{ - retrySQLite([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); - - info->sigs.insert(sigs.begin(), sigs.end()); - - updatePathInfo(*state, *info); - - txn.commit(); - }); -} - - -void LocalStore::signPathInfo(ValidPathInfo & info) -{ - // FIXME: keep secret keys in memory. - - auto secretKeyFiles = settings.secretKeyFiles; - - for (auto & secretKeyFile : secretKeyFiles.get()) { - SecretKey secretKey(readFile(secretKeyFile)); - info.sign(*this, secretKey); - } -} - - -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - - -} From 5526683ad3e259f1c02461c48c7e109de185383d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 18 Jul 2020 07:58:37 +0100 Subject: [PATCH 108/207] fix make's impurity on /bin/sh This is important when using tooling like BEAR to generate compilation database since the used glibc version needs to match for LD_PRELOAD to work. It might be also beneficial when building on systems other than NixOS with nix develop since /bin/sh might be not bash (which is what all nix devs use for testing). This fix is not perfect because Makefile.config.in itself is also build with make but strictly better than the status quo. --- Makefile.config.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.config.in b/Makefile.config.in index b632444e8..5c245b8e9 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -19,6 +19,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ +SHELL = @bash@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ From 3294b0a4b05b8bfa9b8aa9be587dd46a67705864 Mon Sep 17 00:00:00 2001 From: Alex Kovar Date: Sat, 18 Jul 2020 10:23:43 -0500 Subject: [PATCH 109/207] Add newline to profile sourcing line #3393 --- scripts/install-nix-from-closure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 5824c2217..6fb0beb2b 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -207,7 +207,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" fi added=1 break @@ -218,7 +218,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" fi added=1 break From f0100f55909d00f15bfdbef89d6cdcf6c38b2f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 18 Jul 2020 08:48:13 +0100 Subject: [PATCH 110/207] README: improve development docs --- README.md | 25 +++-------------------- doc/manual/hacking.xml | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a1588284d..e5f7a694f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ for more details. On Linux and macOS the easiest way to Install Nix is to run the following shell command (as a user other than root): -``` +```console $ curl -L https://nixos.org/nix/install | sh ``` @@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -### Building Nix - -You can build Nix using one of the targets provided by [release.nix](./release.nix): - -``` -$ nix-build ./release.nix -A build.aarch64-linux -$ nix-build ./release.nix -A build.x86_64-darwin -$ nix-build ./release.nix -A build.i686-linux -$ nix-build ./release.nix -A build.x86_64-linux -``` - -### Development Environment - -You can use the provided `shell.nix` to get a working development environment: - -``` -$ nix-shell -$ ./bootstrap.sh -$ ./configure -$ make -``` +See our [Hacking guide](hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to +build nix from source with nix-build or how to get a development environment. ## Additional Resources diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index b671811d3..c0ece76b6 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -4,18 +4,37 @@ Hacking -This section provides some notes on how to hack on Nix. To get +This section provides some notes on how to hack on Nix. To get the latest version of Nix from GitHub: -$ git clone git://github.com/NixOS/nix.git +$ git clone https://github.com/NixOS/nix.git $ cd nix -To build it and its dependencies: +To build Nix for the current operating system/architecture use + -$ nix-build release.nix -A build.x86_64-linux +$ nix-build + +or if you have a flakes-enabled nix: + + +$ nix build + + +This will build defaultPackage attribute defined in the flake.nix file. + +To build for other platforms add one of the following suffixes to it: aarch64-linux, +i686-linux, x86_64-darwin, x86_64-linux. + +i.e. + + +nix-build -A defaultPackage.x86_64-linux + + To build all dependencies and start a shell in which all @@ -27,13 +46,27 @@ $ nix-shell To build Nix itself in this shell: [nix-shell]$ ./bootstrap.sh -[nix-shell]$ configurePhase -[nix-shell]$ make +[nix-shell]$ ./configure $configureFlags +[nix-shell]$ make -j $NIX_BUILD_CORES To install it in $(pwd)/inst and test it: [nix-shell]$ make install [nix-shell]$ make installcheck +[nix-shell]$ ./inst/bin/nix --version +nix (Nix) 2.4 + + +If you have a flakes-enabled nix you can replace: + + +$ nix-shell + + +by: + + +$ nix develop From ac2fc7ba1fe6b64ec535e4ce63d13fcadf7fdba7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Jul 2020 11:29:46 -0400 Subject: [PATCH 111/207] Apply suggestions from code review Co-authored-by: Eelco Dolstra --- src/libstore/store-api.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5c8dddba5..6c0a61766 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -234,7 +234,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, TeeSink sinkIfNar { narHashSink, caHashSink }; - /* We use the tee sink if we need to hash he nar twice */ + /* We use the tee sink if we need to hash the nar twice */ auto & sink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 ? static_cast(sinkIfNar) : narHashSink; @@ -250,7 +250,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ? fileSink : blank; - parseDump(parseSink, tapped); + parseDump( + parseSink, + method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? *fileSource // don't need to hash twice if we just can use the `narHash` twice + : tapped); auto [narHash, narSize] = narHashSink.finish(); From bf61871271971aa45237fb9ba7fa4c63ae083ff2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Jul 2020 17:42:34 +0000 Subject: [PATCH 112/207] parser.hh -> split.hh --- src/libstore/content-address.cc | 2 +- src/libutil/hash.cc | 2 +- src/libutil/{parser.hh => split.hh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/libutil/{parser.hh => split.hh} (100%) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a562f2d23..749551d1a 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,6 @@ #include "args.hh" #include "content-address.hh" -#include "parser.hh" +#include "split.hh" namespace nix { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index c7ccb809d..65ba1dc81 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -7,7 +7,7 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" -#include "parser.hh" +#include "split.hh" #include "util.hh" #include diff --git a/src/libutil/parser.hh b/src/libutil/split.hh similarity index 100% rename from src/libutil/parser.hh rename to src/libutil/split.hh From c58c6165c554d671f87b463c9ab1d47a5d75bbbb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Jul 2020 17:43:19 +0000 Subject: [PATCH 113/207] Remove period at the end of the exception message --- src/libutil/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 65ba1dc81..dfb3668f1 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -177,7 +177,7 @@ Hash Hash::parseAnyPrefixed(std::string_view original) // Either the string or user must provide the type, if they both do they // must agree. if (!optParsedType) - throw BadHash("hash '%s' does not include a type.", rest); + throw BadHash("hash '%s' does not include a type", rest); return Hash(rest, *optParsedType, isSRI); } From 362ae93851830ecce6ade70462fe991cc522d27b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 20 Jul 2020 14:13:37 -0400 Subject: [PATCH 114/207] Add UnimplementedError to ease grepping for these --- src/libfetchers/git.cc | 2 +- src/libstore/build.cc | 2 +- src/libstore/derivations.hh | 2 +- src/libutil/error.hh | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5d38e0c2b..b1b47c45f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -121,7 +121,7 @@ struct GitInputScheme : InputScheme args.push_back(*ref); } - if (input.getRev()) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented"); args.push_back(destDir); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1c88d91bc..3380dbdaf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1197,7 +1197,7 @@ void DerivationGoal::haveDerivation() if (parsedDrv->contentAddressed()) { settings.requireExperimentalFeature("ca-derivations"); - throw Error("ca-derivations isn't implemented yet"); + throw UnimplementedError("ca-derivations isn't implemented yet"); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 36ac09210..c8f8d10dc 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -44,7 +44,7 @@ struct DerivationOutput /* DEPRECATED: Remove after CA drvs are fully implemented */ StorePath path(const Store & store, std::string_view drvName) const { auto p = pathOpt(store, drvName); - if (!p) throw Error("floating content-addressed derivations are not yet implemented"); + if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented"); return *p; } }; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0daaf3be2..f3babcbde 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -192,6 +192,7 @@ public: MakeError(Error, BaseError); MakeError(UsageError, Error); +MakeError(UnimplementedError, Error); class SysError : public Error { From 6357b1b0fb33bfa26f1d44326fc01d0c86d86f2c Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 20 Jul 2020 14:17:25 -0400 Subject: [PATCH 115/207] Add another Unimplemented case --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9117ff384..e07b33897 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -578,7 +578,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(path, i.first); }, [&](DerivationOutputFloating _) { - throw Error("Floating CA output derivations are not yet implemented"); + throw UnimplementedError("Floating CA output derivations are not yet implemented"); }, }, i.second.output); } From 0ca9744694a5294e995fcddc11f5f195c84036a4 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 20 Jul 2020 15:56:52 -0400 Subject: [PATCH 116/207] Use heuristics to decide when to show the response Due to https://github.com/NixOS/nix/issues/3841 we don't know how print different messages for different verbosity levels. --- src/libstore/filetransfer.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 4d35bd2b5..4149f8155 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -845,8 +845,11 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< : Error(args...), error(error), response(response) { const auto hf = hintfmt(args...); - if (response) { - err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); } else { err.hint = hf; } From 6633605341c6e01bc72b8311e478ed9932719e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 20 Jul 2020 22:30:39 +0100 Subject: [PATCH 117/207] Update doc/manual/hacking.xml Co-authored-by: Eelco Dolstra --- doc/manual/hacking.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index c0ece76b6..d25d4b84a 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -12,7 +12,7 @@ $ cd nix -To build Nix for the current operating system/architecture use +To build Nix for the current operating system/architecture use $ nix-build From 9aae179f34ec2f38167585c07f18a8ab8acefafb Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 20 Jul 2020 20:18:12 -0400 Subject: [PATCH 118/207] Correct bug, thoroughly document addToStoreSlow --- src/libstore/store-api.cc | 62 ++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6c0a61766..14661722d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,40 +222,68 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +/* +The aim of this function is to compute in one pass the correct ValidPathInfo for +the files that we are trying to add to the store. To accomplish that in one +pass, given the different kind of inputs that we can take (normal nar archives, +nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks +and aliases. Also, since the dataflow is obfuscated by this, we include here a +graphviz diagram: + +digraph graphname { + node [shape=box] + fileSource -> narSink + narSink [style=dashed] + narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"] + narSink -> narHashSink [style = dashed, label = "else"] + unsualHashTee -> narHashSink + unsualHashTee -> caHashSink + fileSource -> parseSink + parseSink [style=dashed] + parseSink-> fileSink [style = dashed, label = "Flat"] + parseSink -> blank [style = dashed, label = "Recursive"] + fileSink -> caHashSink +} +*/ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, std::optional expectedCAHash) { - /* FIXME: inefficient: we're reading/hashing 'tmpFile' two - times. */ HashSink narHashSink { htSHA256 }; HashSink caHashSink { hashAlgo }; + + /* Note that fileSink and unusualHashTee must be mutually exclusive, since + they both write to caHashSink. Note that that requisite is currently true + because the former is only used in the flat case. */ RetrieveRegularNARSink fileSink { caHashSink }; + TeeSink unusualHashTee { narHashSink, caHashSink }; - TeeSink sinkIfNar { narHashSink, caHashSink }; - - /* We use the tee sink if we need to hash the nar twice */ - auto & sink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 - ? static_cast(sinkIfNar) + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast(unusualHashTee) : narHashSink; - auto fileSource = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink); + /* Functionally, this means that fileSource will yield the content of + srcPath. The fact that we use scratchpadSink as a temporary buffer here + is an implementation detail. */ + auto fileSource = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); }); - TeeSource tapped { *fileSource, sink }; + /* tapped provides the same data as fileSource, but we also write all the + information to narSink. */ + TeeSource tapped { *fileSource, narSink }; ParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat ? fileSink : blank; - parseDump( - parseSink, - method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 - ? *fileSource // don't need to hash twice if we just can use the `narHash` twice - : tapped); + /* The information that flows from tapped (besides being replicated in + narSink), is now put in parseSink. */ + parseDump(parseSink, tapped); + /* We extract the result of the computation from the sink by calling + finish. */ auto [narHash, narSize] = narHashSink.finish(); auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 @@ -271,8 +299,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { - auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink); + auto source = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); }); addToStore(info, *source); } From 5ce95b9529ad8c53b4395d98635d035d92913091 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Jul 2020 09:47:40 -0400 Subject: [PATCH 119/207] Update src/libstore/build.cc --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 73e29390d..95350356b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3725,7 +3725,7 @@ void DerivationGoal::registerOutputs() DerivationOutputFloating outputHash; std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { - throw Error("No."); + assert(false); // Enclosing `if` handles this case in other branch }, [&](DerivationOutputFixed dof) { outputHash = DerivationOutputFloating { From 922a845ffc4eaa51797bc376d237c6216f0d8391 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 21 Jul 2020 10:24:19 -0400 Subject: [PATCH 120/207] Update chunkSize to the suggested value This was a suggested course of action in a review in one of our earlier commits, https://github.com/NixOS/nix/pull/3801#discussion_r457557079 --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 340fb5306..9ea71170f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1069,7 +1069,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, or the original source is empty */ while (dump.size() < settings.narBufferSize) { auto oldSize = dump.size(); - constexpr size_t chunkSize = 1024; + constexpr size_t chunkSize = 65536; auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; From ae9e9753ce9100ac8ac98fb76d4ac9de45b5734c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= Date: Wed, 22 Jul 2020 13:45:15 +0200 Subject: [PATCH 121/207] README: Fix link to hacking guide The link was previously interpreted as if it were relative to the current file. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e5f7a694f..03c5deb7b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From c56356baccbf006c7835afa9ce58012631e90b31 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 22 Jul 2020 21:37:54 +0000 Subject: [PATCH 122/207] Separate concerns in `scanForReferences` with TeeSink This also will make it easier to use a `HashModuloSink` instead for CA derivations. --- src/libstore/references.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..968e2e425 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -82,7 +79,9 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) PathSet scanForReferences(const string & path, const PathSet & refs, HashResult & hash) { - RefScanSink sink; + RefScanSink refsSink; + HashSink hashSink { htSHA256 }; + TeeSink sink { refsSink, hashSink }; std::map backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,13 +105,13 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); + hash = hashSink.finish(); return found; } From 9423f64ee2b9fe84618e06654fb6b55766b0cf44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 22 Jul 2020 23:59:25 +0000 Subject: [PATCH 123/207] Parse CA derivations using new output variants We no longer need `ParsedDerivation` because everything libstore needs to know about is in the `BasicDerivation` proper. --- src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 1 + src/libexpr/primops.cc | 24 ++++++++++++++++++++++-- src/libstore/build.cc | 2 +- src/libstore/parsed-derivations.cc | 5 ----- src/libstore/parsed-derivations.hh | 2 -- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a2f55504..32b09fb0d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -345,6 +345,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) + , sContentAddressed(symbols.create("__contentAddressed")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 8986952e3..0382298b3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -74,6 +74,7 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sContentAddressed, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d12d571ad..a322a60ed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -583,6 +583,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; + bool contentAddressed = false; std::optional outputHash; std::string outputHashAlgo; auto ingestionMethod = FileIngestionMethod::Flat; @@ -639,6 +640,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->value->type == tNull) continue; } + if (i->name == state.sContentAddressed) + contentAddressed = state.forceBool(*i->value, pos); + /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ if (i->name == state.sArgs) { @@ -694,7 +698,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } catch (Error & e) { - e.addTrace(posDrvName, + e.addTrace(posDrvName, "while evaluating the attribute '%1%' of the derivation '%2%'", key, drvName); throw; @@ -761,7 +765,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); if (outputHash) { - /* Handle fixed-output derivations. */ + /* Handle fixed-output derivations. + + Ignore `__contentAddressed` because fixed output derivations are + already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error({ .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), @@ -783,6 +790,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); } + else if (contentAddressed) { + HashType ht = parseHashType(outputHashAlgo); + for (auto & i : outputs) { + if (!jsonObject) drv.env[i] = hashPlaceholder(i); + drv.outputs.insert_or_assign(i, DerivationOutput { + .output = DerivationOutputFloating { + .method = ingestionMethod, + .hashType = std::move(ht), + }, + }); + } + } + else { /* Compute a hash over the "masked" store derivation, which is the final one except that in the list of outputs, the diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 95350356b..188f50444 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1195,7 +1195,7 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique(drvPath, *drv); - if (parsedDrv->contentAddressed()) { + if (drv->type() == DerivationType::CAFloating) { settings.requireExperimentalFeature("ca-derivations"); throw UnimplementedError("ca-derivations isn't implemented yet"); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c7797b730..24f848e46 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } -bool ParsedDerivation::contentAddressed() const -{ - return getBoolAttr("__contentAddressed", false); -} - } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0b8e8d031..6ee172d81 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -34,8 +34,6 @@ public: bool willBuildLocally() const; bool substitutesAllowed() const; - - bool contentAddressed() const; }; } From b9ead08ca8f3b8e693a4528d0c6f642dcee026fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Thu, 23 Jul 2020 14:21:27 +0200 Subject: [PATCH 124/207] Save changes made by "nix registry pin" to user registry --- src/nix/registry.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 16d7e511f..ebee4545c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -111,6 +111,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); } }; From 2274f63453dc0b5c76a50ae34d6f7c8ef329f411 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Jul 2020 14:34:20 +0000 Subject: [PATCH 125/207] Get rid of `basicDerivation::findOutput` It's a tiny function which is: - hardly worth abstrating over, and also only used once. - doesn't work once we get CA drvs I rewrote the one callsite to be forwards compatable with CA derivations, and also potentially more performant: instead of reading in the derivation it can ust consult the SQLite DB in the common case. --- src/libstore/derivations.cc | 8 -------- src/libstore/derivations.hh | 4 ---- src/nix-env/nix-env.cc | 3 ++- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f325e511a..870901421 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -7,14 +7,6 @@ namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const -{ - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path; -} - bool BasicDerivation::isBuiltin() const { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..00cf1bb08 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -39,10 +39,6 @@ struct BasicDerivation BasicDerivation() { } virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath & findOutput(const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5795c2c09..ddd036070 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -381,7 +381,8 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); + auto outputs = state.store->queryDerivationOutputMap(path); + elem.setOutPath(state.store->printStorePath(outputs.at("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); From 26fcab53e05ce66f5533f0f07ed13d727a891c8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jul 2020 13:40:39 +0200 Subject: [PATCH 126/207] Remove unused file --- doc/manual/expressions/builder-syntax.xml | 119 ---------------------- 1 file changed, 119 deletions(-) delete mode 100644 doc/manual/expressions/builder-syntax.xml diff --git a/doc/manual/expressions/builder-syntax.xml b/doc/manual/expressions/builder-syntax.xml deleted file mode 100644 index e51bade44..000000000 --- a/doc/manual/expressions/builder-syntax.xml +++ /dev/null @@ -1,119 +0,0 @@ -
- -Builder Syntax - -Build script for GNU Hello -(<filename>builder.sh</filename>) - -source $stdenv/setup - -PATH=$perl/bin:$PATH - -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install - - - shows the builder referenced -from Hello's Nix expression (stored in -pkgs/applications/misc/hello/ex-1/builder.sh). -The builder can actually be made a lot shorter by using the -generic builder functions provided by -stdenv, but here we write out the build steps to -elucidate what a builder does. It performs the following -steps: - - - - - - When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the - derivation). For instance, the PATH variable is - emptyActually, it's initialised to - /path-not-set to prevent Bash from setting it - to a default value.. This is done to prevent - undeclared inputs from being used in the build process. If for - example the PATH contained - /usr/bin, then you might accidentally use - /usr/bin/gcc. - - So the first step is to set up the environment. This is - done by calling the setup script of the - standard environment. The environment variable - stdenv points to the location of the standard - environment being used. (It wasn't specified explicitly as an - attribute in , but - mkDerivation adds it automatically.) - - - - - - Since Hello needs Perl, we have to make sure that Perl is in - the PATH. The perl environment - variable points to the location of the Perl package (since it - was passed in as an attribute to the derivation), so - $perl/bin is the - directory containing the Perl interpreter. - - - - - - Now we have to unpack the sources. The - src attribute was bound to the result of - fetching the Hello source tarball from the network, so the - src environment variable points to the location in - the Nix store to which the tarball was downloaded. After - unpacking, we cd to the resulting source - directory. - - The whole build is performed in a temporary directory - created in /tmp, by the way. This directory is - removed after the builder finishes, so there is no need to clean - up the sources afterwards. Also, the temporary directory is - always newly created, so you don't have to worry about files from - previous builds interfering with the current build. - - - - - - GNU Hello is a typical Autoconf-based package, so we first - have to run its configure script. In Nix - every package is stored in a separate location in the Nix store, - for instance - /nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1. - Nix computes this path by cryptographically hashing all attributes - of the derivation. The path is passed to the builder through the - out environment variable. So here we give - configure the parameter - --prefix=$out to cause Hello to be installed in - the expected location. - - - - - - Finally we build Hello (make) and install - it into the location specified by out - (make install). - - - - - -If you are wondering about the absence of error checking on the -result of various commands called in the builder: this is because the -shell script is evaluated with Bash's option, -which causes the script to be aborted if any command fails without an -error check. - -
\ No newline at end of file From e1de1fe0d82d8ba702947dcad3b678cbb9ce9333 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Jul 2020 19:02:57 +0000 Subject: [PATCH 127/207] Make `Buildable` a `std::variant` I think this better captures the intent of what's going on: we either have an opaque store path, or a drv path with some outputs. Having this structure will also help us support CA derivations: we'll have to allow the outpath paths to be optional, so the structure we gain now makes up for the structure we loose then. --- src/libstore/content-address.cc | 4 -- src/libstore/store-api.cc | 4 -- src/libutil/util.hh | 5 ++ src/nix/build.cc | 29 +++++++---- src/nix/command.cc | 17 ++++-- src/nix/installables.cc | 91 ++++++++++++++++----------------- src/nix/installables.hh | 14 +++-- src/nix/log.cc | 13 +++-- 8 files changed, 98 insertions(+), 79 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 6cb69d0a9..f45f77d5c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -24,10 +24,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ab21b0bd5..0f6f3b98f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -865,10 +865,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - bool ValidPathInfo::isContentAddressed(const Store & store) const { if (! ca) return false; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 42130f6dc..630303a5d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -601,4 +601,9 @@ constexpr auto enumerate(T && iterable) } +// C++17 std::visit boilerplate +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + } diff --git a/src/nix/build.cc b/src/nix/build.cc index 0f7e0e123..927301314 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -57,17 +57,24 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile if (dryRun) return; - if (outLink != "") { - for (size_t i = 0; i < buildables.size(); ++i) { - for (auto & output : buildables[i].outputs) - if (auto store2 = store.dynamic_pointer_cast()) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink), true); - } - } - } + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + for (size_t i = 0; i < buildables.size(); ++i) + std::visit(overloaded { + [&](BuildableOpaque bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2->addPermRoot(bo.path, absPath(symlink), true); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink), true); + } + }, + }, buildables[i]); updateProfile(buildables); } diff --git a/src/nix/command.cc b/src/nix/command.cc index af36dda89..04715b43a 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -131,11 +131,18 @@ void MixProfile::updateProfile(const Buildables & buildables) std::optional result; for (auto & buildable : buildables) { - for (auto & output : buildable.outputs) { - if (result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; - } + std::visit(overloaded { + [&](BuildableOpaque bo) { + result = bo.path; + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second; + } + }, + }, buildable); } if (!result) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a13e5a3df..a051950cd 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -307,16 +307,15 @@ struct InstallableStorePath : Installable for (auto & [name, output] : store->readDerivation(storePath).outputs) outputs.emplace(name, output.path); return { - Buildable { + BuildableFromDrv { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - Buildable { - .drvPath = {}, - .outputs = {{"out", storePath}} + BuildableOpaque { + .path = storePath, } }; } @@ -332,33 +331,20 @@ Buildables InstallableValue::toBuildables() { Buildables res; - StorePathSet drvPaths; + std::map drvsToOutputs; + // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { - Buildable b{.drvPath = drv.drvPath}; - drvPaths.insert(drv.drvPath); - auto outputName = drv.outputName; if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - - b.outputs.emplace(outputName, drv.outPath); - - res.push_back(std::move(b)); + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); + drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); } - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = *drvPaths.begin()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; + for (auto & i : drvsToOutputs) + res.push_back(BuildableFromDrv { i.first, i.second }); + + return res; } struct InstallableAttrPath : InstallableValue @@ -653,14 +639,17 @@ Buildables build(ref store, Realise mode, for (auto & i : installables) { for (auto & b : i->toBuildables()) { - if (b.drvPath) { - StringSet outputNames; - for (auto & output : b.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back({*b.drvPath, outputNames}); - } else - for (auto & output : b.outputs) - pathsToBuild.push_back({output.second}); + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path}); + }, + [&](BuildableFromDrv bfd) { + StringSet outputNames; + for (auto & output : bfd.outputs) + outputNames.insert(output.first); + pathsToBuild.push_back({bfd.drvPath, outputNames}); + }, + }, b); buildables.push_back(std::move(b)); } } @@ -681,16 +670,23 @@ StorePathSet toStorePaths(ref store, if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); + std::visit(overloaded { + [&](BuildableOpaque bo) { + outPaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) + outPaths.insert(output.second); + }, + }, b); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; for (auto & i : installables) for (auto & b : i->toBuildables()) - if (b.drvPath) - outPaths.insert(*b.drvPath); + if (auto bfd = std::get_if(&b)) + outPaths.insert(bfd->drvPath); } return outPaths; @@ -714,20 +710,21 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) { - if (!b.drvPath) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - for (auto & output : b.outputs) { - auto derivers = store->queryValidDerivers(output.second); + for (auto & b : i->toBuildables()) + std::visit(overloaded { + [&](BuildableOpaque bo) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", i->what()); + auto derivers = store->queryValidDerivers(bo.path); if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); - } - } else - drvPaths.insert(*b.drvPath); - } + }, + [&](BuildableFromDrv bfd) { + drvPaths.insert(bfd.drvPath); + }, + }, b); return drvPaths; } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index eb34365d4..9edff3331 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -14,12 +14,20 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct Buildable -{ - std::optional drvPath; +struct BuildableOpaque { + StorePath path; +}; + +struct BuildableFromDrv { + StorePath drvPath; std::map outputs; }; +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + typedef std::vector Buildables; struct App diff --git a/src/nix/log.cc b/src/nix/log.cc index 7e10d373a..33380dcf5 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { - auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; - for (auto & output : b.outputs) { - if (log) break; - log = sub->getBuildLog(output.second); - } + auto log = std::visit(overloaded { + [&](BuildableOpaque bo) { + return sub->getBuildLog(bo.path); + }, + [&](BuildableFromDrv bfd) { + return sub->getBuildLog(bfd.drvPath); + }, + }, b); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); From 2292814049256980c6e809ab364ebe0da3c9d76a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 11:19:17 +0200 Subject: [PATCH 128/207] createUnixDomainSocket(): Fix off-by-one error in copying the socket path Reported by Kane York. --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 93798a765..a0a8ff4d3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1581,7 +1581,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (path.size() >= sizeof(addr.sun_path)) + if (path.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%1%' is too long", path); strcpy(addr.sun_path, path.c_str()); From 72f8771094d575e924846f16e5c60742eea9420b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 25 Jul 2020 18:02:42 -0500 Subject: [PATCH 129/207] Allow PRECOMPILE_HEADERS in cross-compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In cross, CXX will look like aarch64-unknown-linux-gnu-g++. We could run some command to check what kind of compiler it is, but for now we can just check if g++ is anywhere in the string. I couldn’t find any "ends with" for makefile, so it can be anywhere in CXX. --- mk/precompiled-headers.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1c0452dc2..500c99e4a 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(CXX), g++) + ifeq ($(findstring g++,$(CXX)), g++) GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch GLOBAL_ORDER_AFTER += $(GCH) - else ifeq ($(CXX), clang++) + else ifeq ($(findstring clang++,$(CXX)), clang++) GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch From 951415b5685fe52d31770eadabd66d95ea75cfae Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Jul 2020 17:56:36 +0000 Subject: [PATCH 130/207] Require `ca-derivations` everywhere we create a CA derivation "create" as in read one in from a serialized form, or build one from scratch in memory. --- src/libexpr/primops.cc | 4 +++- src/libstore/derivations.cc | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bc528140b..784c12b16 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -640,8 +640,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->value->type == tNull) continue; } - if (i->name == state.sContentAddressed) + if (i->name == state.sContentAddressed) { + settings.requireExperimentalFeature("ca-derivations"); contentAddressed = state.forceBool(*i->value, pos); + } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c88bb3c6d..6a12e8734 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -163,12 +163,13 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings }, } } - : DerivationOutput { - .output = DerivationOutputFloating { + : (settings.requireExperimentalFeature("ca-derivations"), + DerivationOutput { + .output = DerivationOutputFloating { .method = std::move(method), .hashType = std::move(hashType), }, - }; + }); } else return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -559,12 +560,13 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) }, } } - : DerivationOutput { + : (settings.requireExperimentalFeature("ca-derivations"), + DerivationOutput { .output = DerivationOutputFloating { .method = std::move(method), .hashType = std::move(hashType), }, - }; + }); } else return DerivationOutput { .output = DerivationOutputInputAddressed { From 8065c6d1606402e936b048aa75ad98ffdd7c8d60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Jul 2020 20:45:34 +0000 Subject: [PATCH 131/207] Abstract out topo sorting logic --- src/libstore/misc.cc | 51 +++++++++++++--------------------------- src/libutil/topo-sort.hh | 40 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 35 deletions(-) create mode 100644 src/libutil/topo-sort.hh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c4d22a634..34a14d30d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "topo-sort.hh" namespace nix { @@ -246,41 +247,21 @@ void Store::queryMissing(const std::vector & targets, StorePaths Store::topoSortPaths(const StorePathSet & paths) { - StorePaths sorted; - StorePathSet visited, parents; - - std::function dfsVisit; - - dfsVisit = [&](const StorePath & path, const StorePath * parent) { - if (parents.count(path)) - throw BuildError("cycle detected in the references of '%s' from '%s'", - printStorePath(path), printStorePath(*parent)); - - if (!visited.insert(path).second) return; - parents.insert(path); - - StorePathSet references; - try { - references = queryPathInfo(path)->references; - } catch (InvalidPath &) { - } - - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) - dfsVisit(i, &path); - - sorted.push_back(path); - parents.erase(path); - }; - - for (auto & i : paths) - dfsVisit(i, nullptr); - - std::reverse(sorted.begin(), sorted.end()); - - return sorted; + return topoSort(paths, + {[&](const StorePath & path) { + StorePathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } + return references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); } diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh new file mode 100644 index 000000000..7a68ff169 --- /dev/null +++ b/src/libutil/topo-sort.hh @@ -0,0 +1,40 @@ +#include "error.hh" + +namespace nix { + +template +std::vector topoSort(std::set items, + std::function(const T &)> getChildren, + std::function makeCycleError) +{ + std::vector sorted; + std::set visited, parents; + + std::function dfsVisit; + + dfsVisit = [&](const T & path, const T * parent) { + if (parents.count(path)) throw makeCycleError(path, *parent); + + if (!visited.insert(path).second) return; + parents.insert(path); + + std::set references = getChildren(path); + + for (auto & i : references) + /* Don't traverse into items that don't exist in our starting set. */ + if (i != path && items.count(i)) + dfsVisit(i, &path); + + sorted.push_back(path); + parents.erase(path); + }; + + for (auto & i : items) + dfsVisit(i, nullptr); + + std::reverse(sorted.begin(), sorted.end()); + + return sorted; +} + +} From f74243846512ffabf082985bca395890c97643e0 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:39:58 +0200 Subject: [PATCH 132/207] Merge legacy `fetchGit`-builtin with the generic `fetchTree`-function The original idea was to implement a git-fetcher in Nix's core that supports content hashes[1]. In #3549[2] it has been suggested to actually use `fetchTree` for this since it's a fairly generic wrapper over the new fetcher-API[3] and already supports content-hashes. This patch implements a new git-fetcher based on `fetchTree` by incorporating the following changes: * Removed the original `fetchGit`-implementation and replaced it with an alias on the `fetchTree` implementation. * Ensured that the `git`-fetcher from `libfetchers` always computes a content-hash and returns an "empty" revision on dirty trees (the latter one is needed to retain backwards-compatibility). * The hash-mismatch error in the fetcher-API exits with code 102 as it usually happens whenever a hash-mismatch is detected by Nix. * Removed the `flakes`-feature-flag: I didn't see a reason why this API is so tightly coupled to the flakes-API and at least `fetchGit` should remain usable without any feature-flags. * It's only possible to specify a `narHash` for a `git`-tree if either a `ref` or a `rev` is given[4]. * It's now possible to specify an URL without a protocol. If it's missing, `file://` is automatically added as it was the case in the original `fetchGit`-implementation. [1] https://github.com/NixOS/nix/pull/3216 [2] https://github.com/NixOS/nix/pull/3549#issuecomment-625194383 [3] https://github.com/NixOS/nix/pull/3459 [4] https://github.com/NixOS/nix/pull/3216#issuecomment-553956703 --- src/libexpr/primops/fetchGit.cc | 91 -------------------------------- src/libexpr/primops/fetchTree.cc | 62 +++++++++++++++++++--- src/libfetchers/fetchers.cc | 2 +- tests/fetchGit.sh | 12 +++-- tests/tarball.sh | 11 ++-- 5 files changed, 73 insertions(+), 105 deletions(-) delete mode 100644 src/libexpr/primops/fetchGit.cc diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc deleted file mode 100644 index 5013e74f0..000000000 --- a/src/libexpr/primops/fetchGit.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "hash.hh" -#include "fetchers.hh" -#include "url.hh" - -namespace nix { - -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional ref; - std::optional rev; - std::string name = "source"; - bool fetchSubmodules = false; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "submodules") - fetchSubmodules = state.forceBool(*attr.value, *attr.pos); - else - throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .errPos = *attr.pos - }); - } - - if (url.empty()) - throw EvalError({ - .hint = hintfmt("'url' argument required"), - .errPos = pos - }); - - } else - url = state.coerceToString(pos, *args[0], context, false, false); - - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); - - if (evalSettings.pureEval && !rev) - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - if (ref) attrs.insert_or_assign("ref", *ref); - if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit{true}); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); - - // FIXME: use name? - auto [tree, input2] = input.fetch(state.store); - - state.mkAttrs(v, 8); - auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - // Backward compatibility: set 'rev' to - // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); - // Backward compatibility: set 'revCount' to 0 for a dirty tree. - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - input2.getRevCount().value_or(0)); - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); - v.attrs->sort(); - - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); -} - -static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); - -} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5f480d919..887d2d14f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -36,6 +36,9 @@ void emitTreeAttrs( mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); } + if (input.getType() == "git") + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); + if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); @@ -48,10 +51,25 @@ void emitTreeAttrs( v.attrs->sort(); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +std::string fixURI(std::string uri, EvalState &state) { - settings.requireExperimentalFeature("flakes"); + state.checkURI(uri); + return uri.find("://") != std::string::npos ? uri : "file://" + uri; +} +void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +{ + string n(name); + attrs.emplace(name, n == "url" ? fixURI(v, state) : v); +} + +static void fetchTree( + EvalState &state, + const Pos &pos, + Value **args, + Value &v, + const std::optional type +) { fetchers::Input input; PathSet context; @@ -64,8 +82,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); + if (attr.value->type == tPath || attr.value->type == tString) + addURI( + state, + attrs, + attr.name, + state.coerceToString(*attr.pos, *attr.value, context, false, false) + ); + else if (attr.value->type == tString) + addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); else if (attr.value->type == tInt) @@ -75,6 +100,9 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V attr.name, showType(*attr.value)); } + if (type) + attrs.emplace("type", type.value()); + if (!attrs.count("type")) throw Error({ .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), @@ -82,8 +110,18 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V }); input = fetchers::Input::fromAttrs(std::move(attrs)); - } else - input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false)); + } else { + auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + + if (type == "git") { + fetchers::Attrs attrs; + attrs.emplace("type", "git"); + attrs.emplace("url", url); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + input = fetchers::Input::fromURL(url); + } + } if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; @@ -99,6 +137,12 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V emitTreeAttrs(state, tree, input2, v); } +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + fetchTree(state, pos, args, v, std::nullopt); +} + static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, @@ -178,7 +222,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args fetch(state, pos, args, v, "fetchTarball", true, "source"); } +static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +{ + fetchTree(state, pos, args, v, "git"); +} + static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); +static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit); } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c3ee9bf43..28db8aa9c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -134,7 +134,7 @@ std::pair Input::fetch(ref store) const if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true)); } diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 9faa5d9f6..1b9364e60 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD) # Fetch a worktree unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] @@ -87,8 +89,6 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] - # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] @@ -102,6 +102,12 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? +[[ "$status" = "102" ]] + +path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath") +[[ $path = $path5 ]] + # tarball-ttl should be ignored if we specify a rev echo delft > $repo/hello git -C $repo add hello @@ -123,7 +129,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --expr "(builtins.fetchGit $repo).rev or null") = null ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a diff --git a/tests/tarball.sh b/tests/tarball.sh index b3ec16d40..88a1a07a0 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -27,10 +27,13 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext From 189e6f5e1d949f50ab0b6e5acd25e230d206692d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:51:16 +0200 Subject: [PATCH 133/207] Bump version to 3.0 Since there are some incompatible changes, it's better to bump the major version number. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 7208c2182..f398a2061 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.4 \ No newline at end of file +3.0 \ No newline at end of file From b8eea7e81af53905be7845dffc6d0a83ea8edc97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2020 13:27:56 +0200 Subject: [PATCH 134/207] Remove putBytes istream->tellg() returns -1 so we can't get the number of bytes written. Fixes 'uploaded 's3://nix-cache/nar/00819r9lp5kajr6baxfw5dhhc0cx8ndxaz43qmd2f0gn1hk1ynlp.nar.xz' (-1 bytes) in 11620 ms' messages. --- src/libstore/s3-binary-cache-store.cc | 7 ++----- src/libstore/s3-binary-cache-store.hh | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085..67935f3ba 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -343,13 +343,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - auto size = istream->tellg(); - - printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, size, duration); + printInfo("uploaded 's3://%s/%s' in %d ms", + bucketName, path, duration); stats.putTimeMs += duration; - stats.putBytes += size; stats.put++; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..b2b75d498 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -19,7 +19,6 @@ public: struct Stats { std::atomic put{0}; - std::atomic putBytes{0}; std::atomic putTimeMs{0}; std::atomic get{0}; std::atomic getBytes{0}; From d564ac1c5086bda75388d21a25583b2c9809f086 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 27 Jul 2020 16:42:02 -0400 Subject: [PATCH 135/207] Offer a safer interface for pathOpt The new interface we offer provides a way of getting all the DerivationOutputs with the storePaths directly, based on the observation that it's the most common usecase. --- src/libstore/derivations.cc | 21 +++++++++++++++++++++ src/libstore/derivations.hh | 19 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c88bb3c6d..944d89fe5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -581,6 +581,27 @@ StringSet BasicDerivation::outputNames() const return names; } +DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const { + DerivationOutputsAndPaths outsAndPaths; + for (auto output : outputs) + outsAndPaths.insert(std::make_pair( + output.first, + std::make_pair(output.second, output.second.path(store, name)) + ) + ); + return outsAndPaths; +} + +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { + DerivationOutputsAndOptPaths outsAndOptPaths; + for (auto output : outputs) + outsAndOptPaths.insert(std::make_pair( + output.first, + std::make_pair(output.second, output.second.pathOpt(store, output.first)) + ) + ); + return outsAndOptPaths; +} std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { auto nameWithSuffix = drvPath.name(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1cda85cb..b0cb870a2 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,6 +41,9 @@ struct DerivationOutput DerivationOutputFloating > output; std::optional hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing + the right derivation name. When in doubt, you should use the safer + interface provided by BasicDerivation::outputsAndPaths */ std::optional pathOpt(const Store & store, std::string_view drvName) const; /* DEPRECATED: Remove after CA drvs are fully implemented */ StorePath path(const Store & store, std::string_view drvName) const { @@ -52,6 +55,15 @@ struct DerivationOutput typedef std::map DerivationOutputs; +/* These are analogues to the previous DerivationOutputs data type, but they + also contains, for each output, the (optional) store path in which it would + be written. To calculate values of these types, see the corresponding + functions in BasicDerivation */ +typedef std::map> + DerivationOutputsAndPaths; +typedef std::map>> + DerivationOutputsAndOptPaths; + /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ typedef std::map DerivationInputs; @@ -101,6 +113,13 @@ struct BasicDerivation /* Return the output names of a derivation. */ StringSet outputNames() const; + /* Calculates the maps that contains all the DerivationOutputs, but + augmented with knowledge of the Store paths they would be written into. + The first one of these functions will be removed when the CA work is + completed */ + DerivationOutputsAndPaths outputsAndPaths(const Store & store) const; + DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + static std::string_view nameFromPath(const StorePath & storePath); }; From 7ef1e3cd1429f336c568ccf4d59cd89cde68efbd Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 28 Jul 2020 13:59:24 -0400 Subject: [PATCH 136/207] Use the new interface --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 8 ++-- src/libstore/build.cc | 84 ++++++++++++++++++------------------- src/libstore/derivations.cc | 16 +++---- src/libstore/local-store.cc | 4 +- src/libstore/misc.cc | 6 +-- src/nix-store/nix-store.cc | 8 ++-- src/nix/installables.cc | 4 +- src/nix/repl.cc | 4 +- src/nix/show-derivation.cc | 6 +-- 10 files changed, 72 insertions(+), 72 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index ec85b844a..7c3416f0b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -303,10 +303,10 @@ SV * derivationFromPath(char * drvPath) hash = newHV(); HV * outputs = newHV(); - for (auto & i : drv.outputs) + for (auto & i : drv.outputsAndPaths(*store())) hv_store( outputs, i.first.c_str(), i.first.size(), - newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0), + newSVpv(store()->printStorePath(i.second.second).c_str(), 0), 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d12d571ad..ee0944fba 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -113,9 +113,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args state.mkList(*outputsVal, drv.outputs.size()); unsigned int outputs_index = 0; - for (const auto & o : drv.outputs) { + for (const auto & o : drv.outputsAndPaths(*state.store)) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -830,9 +830,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * state.mkAttrs(v, 1 + drv.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); - for (auto & i : drv.outputs) { + for (auto & i : drv.outputsAndPaths(*state.store)) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS}); + state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 413fc9c4a..7be60396c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1181,8 +1181,8 @@ void DerivationGoal::haveDerivation() retrySubstitution = false; - for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path(worker.store, drv->name)); + for (auto & i : drv->outputsAndPaths(worker.store)) + worker.store.addTempRoot(i.second.second); /* Check what outputs paths are not already valid. */ auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); @@ -1288,14 +1288,14 @@ void DerivationGoal::repairClosure() /* Get the output closure. */ StorePathSet outputClosure; - for (auto & i : drv->outputs) { + for (auto & i : drv->outputsAndPaths(worker.store)) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure); + worker.store.computeFSClosure(i.second.second, outputClosure); } /* Filter out our own outputs (which we have already checked). */ - for (auto & i : drv->outputs) - outputClosure.erase(i.second.path(worker.store, drv->name)); + for (auto & i : drv->outputsAndPaths(worker.store)) + outputClosure.erase(i.second.second); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1306,8 +1306,8 @@ void DerivationGoal::repairClosure() for (auto & i : inputClosure) if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); - for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i); + for (auto & j : drv.outputsAndPaths(worker.store)) + outputsToDrv.insert_or_assign(j.second.second, i); } /* Check each path (slow!). */ @@ -1466,10 +1466,10 @@ void DerivationGoal::tryToBuild() /* If any of the outputs already exist but are not valid, delete them. */ - for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name))); - deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name))); + for (auto & i : drv->outputsAndPaths(worker.store)) { + if (worker.store.isValidPath(i.second.second)) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.second)); + deletePath(worker.store.Store::toRealPath(i.second.second)); } /* Don't do a remote build if the derivation has the attribute @@ -1919,8 +1919,8 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) for (auto & j : paths2) { if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths); + for (auto & k : drv.outputsAndPaths(worker.store)) + worker.store.computeFSClosure(k.second.second, paths); } } @@ -2014,8 +2014,8 @@ void DerivationGoal::startBuilder() chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ - for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name)); + for (auto & output : drv->outputsAndPaths(worker.store)) + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.second); /* Construct the environment passed to the builder. */ initEnv(); @@ -2199,8 +2199,8 @@ void DerivationGoal::startBuilder() rebuilding a path that is in settings.dirsInChroot (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name))); + for (auto & i : drv->outputsAndPaths(worker.store)) + dirsInChroot.erase(worker.store.printStorePath(i.second.second)); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2612,8 +2612,8 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites); + for (auto & i : drv->outputsAndPaths(worker.store)) + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.second), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2815,9 +2815,9 @@ struct RestrictedStore : public LocalFSStore if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); auto drv = derivationFromPath(path.path); - for (auto & output : drv.outputs) + for (auto & output : drv.outputsAndPaths(*this)) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path(*this, drv.name)); + newPaths.insert(output.second.second); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -3616,8 +3616,8 @@ void DerivationGoal::registerOutputs() to do anything here. */ if (hook) { bool allValid = true; - for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false; + for (auto & i : drv->outputsAndPaths(worker.store)) + if (!worker.store.isValidPath(i.second.second)) allValid = false; if (allValid) return; } @@ -3638,23 +3638,23 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name)); + for (auto & i : drv->outputsAndPaths(worker.store)) referenceablePaths.insert(i.second.second); for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); - if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue; + for (auto & i : drv->outputsAndPaths(worker.store)) { + auto path = worker.store.printStorePath(i.second.second); + if (!missingPaths.count(i.second.second)) continue; Path actualPath = path; if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name)); + auto r = redirectedOutputs.find(i.second.second); if (r != redirectedOutputs.end()) { auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path(worker.store, drv->name)) + && redirectedBadOutputs.count(i.second.second) && pathExists(redirected)) replaceValidPath(path, redirected); if (buildMode == bmCheck) @@ -3721,7 +3721,7 @@ void DerivationGoal::registerOutputs() hash). */ std::optional ca; - if (! std::holds_alternative(i.second.output)) { + if (! std::holds_alternative(i.second.first.output)) { DerivationOutputFloating outputHash; std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { @@ -3736,7 +3736,7 @@ void DerivationGoal::registerOutputs() [&](DerivationOutputFloating dof) { outputHash = dof; }, - }, i.second.output); + }, i.second.first.output); if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ @@ -3753,12 +3753,12 @@ void DerivationGoal::registerOutputs() ? hashPath(outputHash.hashType, actualPath).first : hashFile(outputHash.hashType, actualPath); - auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); + auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.second.name()); // true if either floating CA, or incorrect fixed hash. bool needsMove = true; - if (auto p = std::get_if(& i.second.output)) { + if (auto p = std::get_if(& i.second.first.output)) { Hash & h = p->hash.hash; if (h != h2) { @@ -3919,8 +3919,8 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); + for (auto & i : drv->outputsAndPaths(worker.store)) { + auto path = worker.store.printStorePath(i.second.second); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3937,8 +3937,8 @@ void DerivationGoal::registerOutputs() /* Remove the .check directories if we're done. FIXME: keep them if the result was not determistic? */ if (curRound == nrRounds) { - for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix; + for (auto & i : drv->outputsAndPaths(worker.store)) { + Path prev = worker.store.printStorePath(i.second.second) + checkSuffix; deletePath(prev); } } @@ -4236,12 +4236,12 @@ void DerivationGoal::flushLine() StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { StorePathSet result; - for (auto & i : drv->outputs) { + for (auto & i : drv->outputsAndPaths(worker.store)) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = - worker.store.isValidPath(i.second.path(worker.store, drv->name)) && - (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name))); - if (good == returnValid) result.insert(i.second.path(worker.store, drv->name)); + worker.store.isValidPath(i.second.second) && + (!checkHash || worker.pathContentsGood(i.second.second)); + if (good == returnValid) result.insert(i.second.second); } return result; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 944d89fe5..b9c913788 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -473,12 +473,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); case DerivationType::CAFixed: { std::map outputHashes; - for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.output); + for (const auto & i : drv.outputsAndPaths(store)) { + auto & dof = std::get(i.second.first.output); auto hash = hashString(htSHA256, "fixed:out:" + dof.hash.printMethodAlgo() + ":" + dof.hash.hash.to_string(Base16, false) + ":" - + store.printStorePath(i.second.path(store, drv.name))); + + store.printStorePath(i.second.second)); outputHashes.insert_or_assign(i.first, std::move(hash)); } return outputHashes; @@ -532,8 +532,8 @@ bool wantOutput(const string & output, const std::set & wanted) StorePathSet BasicDerivation::outputPaths(const Store & store) const { StorePathSet paths; - for (auto & i : outputs) - paths.insert(i.second.path(store, name)); + for (auto & i : outputsAndPaths(store)) + paths.insert(i.second.second); return paths; } @@ -642,9 +642,9 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); - for (auto & i : drv.outputs) { + for (auto & i : drv.outputsAndPaths(store)) { out << i.first - << store.printStorePath(i.second.path(store, drv.name)); + << store.printStorePath(i.second.second); std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { out << "" << ""; @@ -657,7 +657,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, - }, i.second.output); + }, i.second.first.output); } writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0ca21bc40..a87dde1c5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -618,11 +618,11 @@ uint64_t LocalStore::addValidPath(State & state, registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - for (auto & i : drv.outputs) { + for (auto & i : drv.outputsAndPaths(*this)) { state.stmtAddDerivationOutput.use() (id) (i.first) - (printStorePath(i.second.path(*this, drv.name))) + (printStorePath(i.second.second)) .exec(); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c4d22a634..46553fecb 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -196,10 +196,10 @@ void Store::queryMissing(const std::vector & targets, ParsedDerivation parsedDrv(StorePath(path.path), *drv); PathSet invalid; - for (auto & j : drv->outputs) + for (auto & j : drv->outputsAndPaths(*this)) if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path(*this, drv->name))) - invalid.insert(printStorePath(j.second.path(*this, drv->name))); + && !isValidPath(j.second.second)) + invalid.insert(printStorePath(j.second.second)); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9c8874fd4..87888a1d0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -218,8 +218,8 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, if (useOutput && storePath.isDerivation()) { auto drv = store->derivationFromPath(storePath); StorePathSet outputs; - for (auto & i : drv.outputs) - outputs.insert(i.second.path(*store, drv.name)); + for (auto & i : drv.outputsAndPaths(*store)) + outputs.insert(i.second.second); return outputs; } else return {storePath}; @@ -312,8 +312,8 @@ static void opQuery(Strings opFlags, Strings opArgs) auto i2 = store->followLinksToStorePath(i); if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); - for (auto & j : drv.outputs) - cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name))); + for (auto & j : drv.outputsAndPaths(*store)) + cout << fmt("%1%\n", store->printStorePath(j.second.second)); } break; } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b245e417e..41c077c47 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -305,8 +305,8 @@ struct InstallableStorePath : Installable if (storePath.isDerivation()) { std::map outputs; auto drv = store->readDerivation(storePath); - for (auto & [name, output] : drv.outputs) - outputs.emplace(name, output.path(*store, drv.name)); + for (auto & i : drv.outputsAndPaths(*store)) + outputs.emplace(i.first, i.second.second); return { Buildable { .drvPath = storePath, diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fb9050d0d..803d89018 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -485,8 +485,8 @@ bool NixRepl::processLine(string line) if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath))); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; - for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); + for (auto & i : drv.outputsAndPaths(*state->store)) + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.second)); } } else if (command == ":i") { runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 25ea19834..87122be87 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -67,9 +67,9 @@ struct CmdShowDerivation : InstallablesCommand { auto outputsObj(drvObj.object("outputs")); - for (auto & output : drv.outputs) { + for (auto & output : drv.outputsAndPaths(*store)) { auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); + outputObj.attr("path", store->printStorePath(output.second.second)); std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { @@ -81,7 +81,7 @@ struct CmdShowDerivation : InstallablesCommand [&](DerivationOutputFloating dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, - }, output.second.output); + }, output.second.first.output); } } From 2980b244b7e5f1660c4a07d7589f4a2dd47f9acd Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 28 Jul 2020 15:39:45 -0400 Subject: [PATCH 137/207] Use assert(false) instead of abort() --- src/libstore/derivations.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b708ecc57..ca77366bf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -14,7 +14,7 @@ bool derivationIsCA(DerivationType dt) { }; // Since enums can have non-variant values, but making a `default:` would // disable exhaustiveness warnings. - abort(); + assert(false); } bool derivationIsFixed(DerivationType dt) { @@ -22,7 +22,7 @@ bool derivationIsFixed(DerivationType dt) { case DerivationType::Regular: return false; case DerivationType::CAFixed: return true; }; - abort(); + assert(false); } bool derivationIsImpure(DerivationType dt) { @@ -30,7 +30,7 @@ bool derivationIsImpure(DerivationType dt) { case DerivationType::Regular: return false; case DerivationType::CAFixed: return true; }; - abort(); + assert(false); } // FIXME Put this somewhere? From e4940e90f399c5cde5f44f099ca5cdf5340d22b8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 28 Jul 2020 22:46:39 +0200 Subject: [PATCH 138/207] Restore backwards-compat for current `builtins.fetchGit` If a repo is dirty, it used to return a `rev` object with an "empty" sha1 (0000000000000000000000000000000000000000). Please note that this only applies for `builtins.fetchGit` and *not* for `builtins.fetchTree{ type = "git"; }`. --- src/libexpr/flake/flake.hh | 2 +- src/libexpr/primops/fetchTree.cc | 17 +++++++++++++---- tests/fetchGit.sh | 4 +++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 77f3abdeb..c2bb2888b 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -106,6 +106,6 @@ void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, const fetchers::Input & input, - Value & v); + Value & v, bool emptyRevFallback = false); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 887d2d14f..cddcf0e59 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -14,7 +14,8 @@ void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, const fetchers::Input & input, - Value & v) + Value & v, + bool emptyRevFallback) { assert(input.isImmutable()); @@ -34,6 +35,11 @@ void emitTreeAttrs( if (auto rev = input.getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); } if (input.getType() == "git") @@ -41,6 +47,8 @@ void emitTreeAttrs( if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); if (auto lastModified = input.getLastModified()) { mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); @@ -68,7 +76,8 @@ static void fetchTree( const Pos &pos, Value **args, Value &v, - const std::optional type + const std::optional type, + bool emptyRevFallback = false ) { fetchers::Input input; PathSet context; @@ -134,7 +143,7 @@ static void fetchTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v); + emitTreeAttrs(state, tree, input2, v, emptyRevFallback); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -224,7 +233,7 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) { - fetchTree(state, pos, args, v, "git"); + fetchTree(state, pos, args, v, "git", true); } static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 1b9364e60..cedd796f7 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -89,6 +89,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] @@ -129,7 +131,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --impure --expr "(builtins.fetchGit $repo).rev or null") = null ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a From c159f48a39835d5b2fe7c1ddd4467bc093ee251f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 00:24:55 +0200 Subject: [PATCH 139/207] Cleanup --- src/libstore/store-api.cc | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b6e6952b7..c804399d2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -729,9 +729,9 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); if (missing.empty()) return; @@ -748,29 +748,27 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st ThreadPool pool; - processGraph(pool, - PathSet(missing.begin(), missing.end()), + processGraph(pool, + StorePathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const StorePath & storePath) { + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); + auto info = srcStore->queryPathInfo(storePath); bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); - if (!dstStore->isValidPath(storePath)) { MaintainCount mc(nrRunning); showProgress(); @@ -780,7 +778,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } From 4c0077a07d2ee5362bd8ffb0ea9ee053c759303d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 00:48:39 +0200 Subject: [PATCH 140/207] Fix RemoteStore::addToStore() latency Since 6185d25e523a3cd223dd6f6aca10cf6ff15b4823, this was very latency-bound since it required a round-trip for every 32 KiB. So for example copying a 514 MiB closure over a virtual ethernet device with a articial delay of just 1 ms took 343s. Now it takes 2.7s. Fixes #3372. --- src/libstore/daemon.cc | 90 +++++++++++++++++++++++++++------ src/libstore/remote-store.cc | 81 +++++++++++++++++++++++++++-- src/libstore/worker-protocol.hh | 2 +- 3 files changed, 154 insertions(+), 19 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 503e04f92..478ae39ca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -86,7 +86,7 @@ struct TunnelLogger : public Logger } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -703,24 +703,84 @@ static void performOp(TunnelLogger * logger, ref store, if (!trusted) info.ultimate = false; - std::unique_ptr source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); - else { - StringSink saved; - TeeSource tee { from, saved }; - ParseSink ether; - parseDump(ether, tee); - source = std::make_unique(std::move(*saved.s)); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + + struct FramedSource : Source + { + Source & from; + bool eof = false; + std::vector pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } + }; + + logger->startWork(); + + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8d01c6667..4ed81d9d8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,9 +503,84 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + + std::exception_ptr ex; + + struct FramedSink : BufferedSink + { + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; + }; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + conn.processStderr(0, nullptr); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(conn, ex); + copyNAR(source, sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 8b538f6da..dcba73116 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x116 +#define PROTOCOL_VERSION 0x117 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From fa8515d7ec160938b0287f405a43aeeca971777e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 29 Jul 2020 00:57:20 -0500 Subject: [PATCH 141/207] Set regex_constants::match_continuous for quicker search match_continuous limits the search to the current start position, instead of searching the entire file. On libc++, this improves performance dramatically: $ time /nix/store/70ai68dfm6xbzwn26j5n4li9di52ylia-nix-3.0pre20200728_c159f48/bin/nix print-dev-env >/dev/null /nix/store/70ai68dfm6xbzwn26j5n4li9di52ylia-nix-3.0pre20200728_c159f48/bin/ni 2.39s user 0.19s system 64% cpu 4.032 total $ time /nix/store/cwjfxxlp83zln4mfyy1d2dbsx7f6s962-nix-3.0pre20200728_dirty/bin/nix print-dev-env >/dev/null /nix/store/cwjfxxlp83zln4mfyy1d2dbsx7f6s962-nix-3.0pre20200728_dirty/bin/nix 0.09s user 0.05s system 65% cpu 0.204 total Fixes #3874 --- src/nix/develop.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 2f590f53f..a0c119e43 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path) std::smatch match; - if (std::regex_search(pos, file.cend(), match, declareRegex)) { + if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { pos = match[0].second; exported.insert(match[1]); } - else if (std::regex_search(pos, file.cend(), match, varRegex)) { + else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { res.bashFunctions = std::string(pos, file.cend()); break; } From f63839bfa402f73e001a5ee412c3f6a6b95d1a38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 18:04:35 +0200 Subject: [PATCH 142/207] Cleanup --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4ed81d9d8..616081aa1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -545,7 +545,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, std::thread stderrThread([&]() { try { - conn.processStderr(0, nullptr); + conn.processStderr(); } catch (...) { ex = std::current_exception(); } From e4f6f8da77f8de0f19e81f1cbb60052ca92dd371 Mon Sep 17 00:00:00 2001 From: Mat Marini Date: Wed, 29 Jul 2020 20:50:06 -0400 Subject: [PATCH 143/207] Prevent nix-build from accepting --packages --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f77de56ea..d8f2fe6d7 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -192,7 +192,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (*arg == "--packages" || *arg == "-p") + else if (runEnv && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { From d9e23bfee2f1995b3047d67e578e2cfd4e94dea6 Mon Sep 17 00:00:00 2001 From: Mat Marini Date: Wed, 29 Jul 2020 20:53:02 -0400 Subject: [PATCH 144/207] Prevent nix-build from accepting --run/--command --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d8f2fe6d7..723db71fd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -174,7 +174,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--run-env") // obsolete runEnv = true; - else if (*arg == "--command" || *arg == "--run") { + else if (runEnv && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; From ebee2b7852a8c0441e9470a0df8ec3c71d322109 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 13:00:30 +0200 Subject: [PATCH 145/207] receiveContents(): unsigned int -> size_t --- src/libstore/nar-accessor.cc | 2 +- src/libutil/archive.cc | 2 +- src/libutil/archive.hh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index d884a131e..e438b5710 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -86,7 +86,7 @@ struct NarAccessor : public FSAccessor parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override + void receiveContents(unsigned char * data, size_t len) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51c88537e..c07e710cd 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -338,7 +338,7 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { writeFull(fd.get(), data, len); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 57780d16a..b6e1188c4 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -58,7 +58,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; + virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; @@ -77,7 +77,7 @@ struct RetrieveRegularNARSink : ParseSink regular = false; } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { sink(data, len); } From 3f6e88a5527dcc4d58e3147f78388a88eb8896e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 13:10:49 +0200 Subject: [PATCH 146/207] unsigned long long -> uint64_t --- src/build-remote/build-remote.cc | 8 ++++---- src/libexpr/primops.cc | 2 +- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 4 ++-- src/libstore/build.cc | 10 +++++----- src/libstore/daemon.cc | 2 +- src/libstore/gc.cc | 14 +++++++------- src/libstore/local-store.hh | 4 ++-- src/libstore/misc.cc | 6 +++--- src/libstore/nar-accessor.cc | 2 +- src/libstore/optimise-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 10 +++++----- src/libutil/archive.cc | 8 ++++---- src/libutil/archive.hh | 2 +- src/libutil/config.cc | 2 +- src/libutil/hash.hh | 4 ++-- src/libutil/json.cc | 2 +- src/libutil/serialise.hh | 16 ++++++++-------- src/libutil/util.cc | 8 ++++---- src/libutil/util.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 6 ++---- src/nix-store/nix-store.cc | 10 ++++------ src/nix/path-info.cc | 2 +- 26 files changed, 67 insertions(+), 71 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e07117496..3579d8fff 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -33,7 +33,7 @@ std::string escapeUri(std::string uri) static string currentLoad; -static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) +static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } @@ -119,7 +119,7 @@ static int _main(int argc, char * * argv) bool rightType = false; Machine * bestMachine = nullptr; - unsigned long long bestLoad = 0; + uint64_t bestLoad = 0; for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); @@ -130,8 +130,8 @@ static int _main(int argc, char * * argv) m.mandatoryMet(requiredFeatures)) { rightType = true; AutoCloseFD free; - unsigned long long load = 0; - for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + uint64_t load = 0; + for (uint64_t slot = 0; slot < m.maxJobs; ++slot) { auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5059286b3..a98cadc80 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -65,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(drvs); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52718c231..2b1f25ca3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() void printMissing(ref store, const std::vector & paths, Verbosity lvl) { - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); @@ -45,7 +45,7 @@ void printMissing(ref store, const std::vector & pa void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) + uint64_t downloadSize, uint64_t narSize, Verbosity lvl) { if (!willBuild.empty()) { if (willBuild.size() == 1) @@ -384,7 +384,7 @@ RunPager::~RunPager() } -string showBytes(unsigned long long bytes) +string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f558247c0..ffae5d796 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -47,7 +47,7 @@ void printMissing( void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); + uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end); @@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt; /* GC helpers. */ -string showBytes(unsigned long long bytes); +string showBytes(uint64_t bytes); struct GCResults; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d26f96b74..05235510d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -5038,7 +5038,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 478ae39ca..9ac7e7cca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -790,7 +790,7 @@ static void performOp(TunnelLogger * logger, ref store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); writeStorePaths(*store, to, willBuild); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index aaed5c218..e74382ed2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -500,7 +500,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 355c2814f..547fe589c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -29,8 +29,8 @@ struct Derivation; struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c4d22a634..a542a259d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -110,7 +110,7 @@ void Store::computeFSClosure(const StorePath & startPath, void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +122,8 @@ void Store::queryMissing(const std::vector & targets, { std::unordered_set done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index e438b5710..59ec164b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -79,7 +79,7 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); parents.top()->size = (uint64_t) size; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..e4b4b6213 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) +static string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 616081aa1..611f70783 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -782,7 +782,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c1b78b6a..eb6f225b1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -94,7 +94,7 @@ public: void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a7288d0cc..667dc4117 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -85,7 +85,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits::max()}; + uint64_t maxFreed{std::numeric_limits::max()}; }; @@ -97,7 +97,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; @@ -105,8 +105,8 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; @@ -610,7 +610,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index c07e710cd..ce7cf9754 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -150,17 +150,17 @@ static void skipGeneric(Source & source) static void parseContents(ParseSink & sink, Source & source, const Path & path) { - unsigned long long size = readLongLong(source); + uint64_t size = readLongLong(source); sink.preallocateContents(size); - unsigned long long left = size; + uint64_t left = size; std::vector buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); - if ((unsigned long long)n > left) n = left; + if ((uint64_t)n > left) n = left; source(buf.data(), n); sink.receiveContents(buf.data(), n); left -= n; @@ -323,7 +323,7 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(unsigned long long len) + void preallocateContents(uint64_t len) { #if HAVE_POSIX_FALLOCATE if (len) { diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index b6e1188c4..5665732d2 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -57,7 +57,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; + virtual void preallocateContents(uint64_t size) { }; virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8fc700a2b..8454037f2 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -284,7 +284,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; -template class BaseSetting; +template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 98ee1bed0..abcd58f24 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -111,7 +111,7 @@ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -typedef std::pair HashResult; +typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); @@ -141,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink private: HashType ht; Ctx * ctx; - unsigned long long bytes; + uint64_t bytes; public: HashSink(HashType ht); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 01331947e..104b99e6d 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -29,7 +29,7 @@ template<> void toJSON(std::ostream & str, const unsigned int & n) template<> void toJSON(std::ostream & str, const long & n) { str << n; } template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const uint64_t & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5d9acf887..c29c6b29b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -312,14 +312,14 @@ T readNum(Source & source) source(buf, sizeof(buf)); uint64_t n = - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); + ((uint64_t) buf[0]) | + ((uint64_t) buf[1] << 8) | + ((uint64_t) buf[2] << 16) | + ((uint64_t) buf[3] << 24) | + ((uint64_t) buf[4] << 32) | + ((uint64_t) buf[5] << 40) | + ((uint64_t) buf[6] << 48) | + ((uint64_t) buf[7] << 56); if (n > std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a0a8ff4d3..97d278581 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -374,7 +374,7 @@ void writeLine(int fd, string s) } -static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) { checkInterrupt(); @@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by } } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(const Path & path, uint64_t & bytesFreed) { Path dir = dirOf(path); if (dir == "") @@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) void deletePath(const Path & path) { - unsigned long long dummy; + uint64_t dummy; deletePath(path, dummy); } -void deletePath(const Path & path, unsigned long long & bytesFreed) +void deletePath(const Path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); bytesFreed = 0; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 42130f6dc..6850b5a7a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -125,7 +125,7 @@ void writeLine(int fd, string s); second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); -void deletePath(const Path & path, unsigned long long & bytesFreed); +void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 723db71fd..94412042f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -325,7 +325,7 @@ static void _main(int argc, char * * argv) auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index aa5ada3a6..bcf1d8518 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -67,10 +67,8 @@ static int _main(int argc, char * * argv) deleteOlderThan = getArg(*arg, arg, end); } else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--max-freed") { - long long maxFreed = getIntArg(*arg, arg, end, true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*arg == "--max-freed") + options.maxFreed = std::max(getIntArg(*arg, arg, end, true), (int64_t) 0); else return false; return true; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 76bf5154a..7b26970ef 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.push_back(store->followLinksToStorePathWithOutputs(i)); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*i == "--max-freed") + options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0); else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) if (!path.isDerivation()) paths2.push_back({path}); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 65f73cd94..0c12efaf0 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } - void printSize(unsigned long long value) + void printSize(uint64_t value) { if (!humanReadable) { std::cout << fmt("\t%11d", value); From a785b3eddf8c02750b1715939069d20980bd5125 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 15:27:28 +0200 Subject: [PATCH 147/207] Fix build --- src/libutil/config.cc | 2 +- src/libutil/json.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8454037f2..8fc700a2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -284,7 +284,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; -template class BaseSetting; +template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 104b99e6d..01331947e 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -29,7 +29,7 @@ template<> void toJSON(std::ostream & str, const unsigned int & n) template<> void toJSON(std::ostream & str, const long & n) { str << n; } template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const uint64_t & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } From 2f4250a4167f2b1c10e5fa9c6163f84bb9bbd740 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 11:33:22 -0500 Subject: [PATCH 148/207] Add "export" to Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a ‘nix export’ command which hooks into nix-bundle. It can be used in a similar way as nix-bundle, with the benefit of hooking into the new “app” functionality. For instance, $ nix export nixpkgs#jq $ ./jq --help jq - commandline JSON processor [version 1.6] ... $ scp jq machine-without-nix: $ ssh machine-without-nix ./jq jq - commandline JSON processor [version 1.6] ... Note that nix-bundle currently requires Linux to run. Other exporters might not have that requirement. “exporters” are meant to be reusable, so that, other repos can implement their own bundling. Fixes #3705 --- src/nix/export.cc | 126 ++++++++++++++++++++++++++++++++++++++++++++++ src/nix/flake.cc | 25 +++++++++ 2 files changed, 151 insertions(+) create mode 100644 src/nix/export.cc diff --git a/src/nix/export.cc b/src/nix/export.cc new file mode 100644 index 000000000..9e7816605 --- /dev/null +++ b/src/nix/export.cc @@ -0,0 +1,126 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "fs-accessor.hh" + +using namespace nix; + +struct CmdExport : InstallableCommand +{ + std::string exporter = "github:matthewbauer/nix-bundle"; + Path outLink; + + CmdExport() + { + addFlag({ + .longName = "exporter", + .description = "use custom exporter", + .labels = {"flake-url"}, + .handler = {&exporter}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + } + + std::string description() override + { + return "export an application out of the Nix store"; + } + + Examples examples() override + { + return { + Example{ + "To export Hello:", + "nix export hello" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + + auto app = installable->toApp(*evalState); + store->buildPaths(app.context); + + auto [exporterFlakeRef, exporterName] = parseFlakeRefWithFragment(exporter, absPath(".")); + const flake::LockFlags lockFlags{ .writeLockFile = false }; + auto exporter = InstallableFlake( + evalState, std::move(exporterFlakeRef), + Strings{exporterName == "" ? ("defaultExporter." + settings.thisSystem.get()) : exporterName}, + Strings({"exporters." + settings.thisSystem.get() + "."}), lockFlags); + + Value * arg = evalState->allocValue(); + evalState->mkAttrs(*arg, 1); + + PathSet context; + for (auto & i : app.context) + context.insert("=" + store->printStorePath(i.path)); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + + auto vRes = evalState->allocValue(); + evalState->callFunction(*exporter.toValue(*evalState).first, *arg, *vRes, noPos); + + if (!evalState->isDerivation(*vRes)) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); + if (i == vRes->attrs->end()) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + + i = vRes->attrs->find(evalState->sOutPath); + if (i == vRes->attrs->end()) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + + store->buildPaths({{drvPath}}); + + auto accessor = store->getFSAccessor(); + auto outPathS = store->printStorePath(outPath); + if (accessor->stat(outPathS).type != FSAccessor::tRegular) + throw Error("'%s' is not a file; an exporter must only create a single file", outPathS); + + auto info = store->queryPathInfo(outPath); + if (!info->references.empty()) + throw Error("'%s' has references; an exporter must not leave any references", outPathS); + + if (outLink == "") + outLink = baseNameOf(app.program); + + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(outLink), true); + } +}; + +static auto r2 = registerCommand("export"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 027a9871e..db3ebdc2a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -368,6 +368,21 @@ struct CmdFlakeCheck : FlakeCommand } }; + auto checkExporter = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda) + throw Error("exporter must be a function"); + if (!v.lambda.fun->formals || + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("args")) == v.lambda.fun->formals->argNames.end()) + throw Error("exporter must take formal arguments 'program' and 'args'"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -490,6 +505,16 @@ struct CmdFlakeCheck : FlakeCommand *attr.value, *attr.pos); } + else if (name == "defaultExporter") + checkExporter(name, vOutput, pos); + + else if (name == "exporters") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkExporter(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + else warn("unknown flake output '%s'", name); From 5d04a4db9b3d8cbfbaacbc48a587d21d90844871 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 11:45:47 -0500 Subject: [PATCH 149/207] Handle exporters checking correctly --- src/nix/flake.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index db3ebdc2a..cf7e4b7f5 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -374,9 +374,8 @@ struct CmdFlakeCheck : FlakeCommand if (v.type != tLambda) throw Error("exporter must be a function"); if (!v.lambda.fun->formals || - v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || - v.lambda.fun->formals->argNames.find(state->symbols.create("args")) == v.lambda.fun->formals->argNames.end()) - throw Error("exporter must take formal arguments 'program' and 'args'"); + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) + throw Error("exporter must take formal argument 'program'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -506,13 +505,21 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultExporter") - checkExporter(name, vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkExporter(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); + } else if (name == "exporters") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - checkExporter(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkExporter( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } } else From 52407f83a133ba605a1f0891c3812fa89038bba8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:03:57 -0500 Subject: [PATCH 150/207] exporter -> bundler --- src/nix/{export.cc => bundle.cc} | 42 ++++++++++++++++---------------- src/nix/flake.cc | 14 +++++------ 2 files changed, 28 insertions(+), 28 deletions(-) rename src/nix/{export.cc => bundle.cc} (67%) diff --git a/src/nix/export.cc b/src/nix/bundle.cc similarity index 67% rename from src/nix/export.cc rename to src/nix/bundle.cc index 9e7816605..e7338262b 100644 --- a/src/nix/export.cc +++ b/src/nix/bundle.cc @@ -6,18 +6,18 @@ using namespace nix; -struct CmdExport : InstallableCommand +struct CmdBundle : InstallableCommand { - std::string exporter = "github:matthewbauer/nix-bundle"; + std::string bundler = "github:matthewbauer/nix-bundle"; Path outLink; - CmdExport() + CmdBundle() { addFlag({ - .longName = "exporter", - .description = "use custom exporter", + .longName = "bundler", + .description = "use custom bundler", .labels = {"flake-url"}, - .handler = {&exporter}, + .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { completeFlakeRef(getStore(), prefix); }} @@ -35,15 +35,15 @@ struct CmdExport : InstallableCommand std::string description() override { - return "export an application out of the Nix store"; + return "bundle an application so that it works outside of the Nix store"; } Examples examples() override { return { Example{ - "To export Hello:", - "nix export hello" + "To bundle Hello:", + "nix bundle hello" }, }; } @@ -71,12 +71,12 @@ struct CmdExport : InstallableCommand auto app = installable->toApp(*evalState); store->buildPaths(app.context); - auto [exporterFlakeRef, exporterName] = parseFlakeRefWithFragment(exporter, absPath(".")); + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; - auto exporter = InstallableFlake( - evalState, std::move(exporterFlakeRef), - Strings{exporterName == "" ? ("defaultExporter." + settings.thisSystem.get()) : exporterName}, - Strings({"exporters." + settings.thisSystem.get() + "."}), lockFlags); + auto bundler = InstallableFlake( + evalState, std::move(bundlerFlakeRef), + Strings{bundlerName == "" ? ("defaultBundler." + settings.thisSystem.get()) : bundlerName}, + Strings({"bundlers." + settings.thisSystem.get() + "."}), lockFlags); Value * arg = evalState->allocValue(); evalState->mkAttrs(*arg, 1); @@ -87,21 +87,21 @@ struct CmdExport : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); auto vRes = evalState->allocValue(); - evalState->callFunction(*exporter.toValue(*evalState).first, *arg, *vRes, noPos); + evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); if (!evalState->isDerivation(*vRes)) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); if (i == vRes->attrs->end()) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a bderivation", bundler.what()); PathSet context2; StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); i = vRes->attrs->find(evalState->sOutPath); if (i == vRes->attrs->end()) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); @@ -110,11 +110,11 @@ struct CmdExport : InstallableCommand auto accessor = store->getFSAccessor(); auto outPathS = store->printStorePath(outPath); if (accessor->stat(outPathS).type != FSAccessor::tRegular) - throw Error("'%s' is not a file; an exporter must only create a single file", outPathS); + throw Error("'%s' is not a file; a bundler must only create a single file", outPathS); auto info = store->queryPathInfo(outPath); if (!info->references.empty()) - throw Error("'%s' has references; an exporter must not leave any references", outPathS); + throw Error("'%s' has references; a bundler must not leave any references", outPathS); if (outLink == "") outLink = baseNameOf(app.program); @@ -123,4 +123,4 @@ struct CmdExport : InstallableCommand } }; -static auto r2 = registerCommand("export"); +static auto r2 = registerCommand("bundle"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cf7e4b7f5..1190d7997 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -368,14 +368,14 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkExporter = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); if (v.type != tLambda) - throw Error("exporter must be a function"); + throw Error("bundler must be a function"); if (!v.lambda.fun->formals || v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) - throw Error("exporter must take formal argument 'program'"); + throw Error("bundler must take formal argument 'program'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -504,19 +504,19 @@ struct CmdFlakeCheck : FlakeCommand *attr.value, *attr.pos); } - else if (name == "defaultExporter") + else if (name == "defaultBundler") for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); - checkExporter(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); + checkBundler(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); } - else if (name == "exporters") { + else if (name == "bundlers") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); state->forceAttrs(*attr.value, *attr.pos); for (auto & attr2 : *attr.value->attrs) - checkExporter( + checkBundler( fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); } From 1a705637ce96c30424409d74cb3411554ae2847e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:05:51 -0500 Subject: [PATCH 151/207] Remove single file restriction for bundler --- src/nix/bundle.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e7338262b..677d42520 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -107,10 +107,7 @@ struct CmdBundle : InstallableCommand store->buildPaths({{drvPath}}); - auto accessor = store->getFSAccessor(); auto outPathS = store->printStorePath(outPath); - if (accessor->stat(outPathS).type != FSAccessor::tRegular) - throw Error("'%s' is not a file; a bundler must only create a single file", outPathS); auto info = store->queryPathInfo(outPath); if (!info->references.empty()) From 22fcfdf18af84bbbbc1bd416a9eed37a64e26eb5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:18:48 -0500 Subject: [PATCH 152/207] Address misc review --- src/nix/bundle.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 677d42520..95aed5269 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -9,7 +9,7 @@ using namespace nix; struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; - Path outLink; + std::optional outLink; CmdBundle() { @@ -92,18 +92,18 @@ struct CmdBundle : InstallableCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); - if (i == vRes->attrs->end()) - throw Error("the bundler '%s' does not produce a bderivation", bundler.what()); - - PathSet context2; - StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); - - i = vRes->attrs->find(evalState->sOutPath); - if (i == vRes->attrs->end()) + auto attr1 = vRes->attrs->find(evalState->sDrvPath); + if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + + auto attr2 = vRes->attrs->find(evalState->sOutPath); + if (!attr2) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); store->buildPaths({{drvPath}}); @@ -113,10 +113,10 @@ struct CmdBundle : InstallableCommand if (!info->references.empty()) throw Error("'%s' has references; a bundler must not leave any references", outPathS); - if (outLink == "") + if (!outLink) outLink = baseNameOf(app.program); - store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(outLink), true); + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink), true); } }; From fa2d1fb36e2ee92622379b9716f5b06e73aae72e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:37:05 -0500 Subject: [PATCH 153/207] Pass system to bundler --- src/nix/bundle.cc | 10 +++++++--- src/nix/flake.cc | 21 +++++++-------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 95aed5269..f2b78eea5 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -75,17 +75,21 @@ struct CmdBundle : InstallableCommand const flake::LockFlags lockFlags{ .writeLockFile = false }; auto bundler = InstallableFlake( evalState, std::move(bundlerFlakeRef), - Strings{bundlerName == "" ? ("defaultBundler." + settings.thisSystem.get()) : bundlerName}, - Strings({"bundlers." + settings.thisSystem.get() + "."}), lockFlags); + Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings({"bundlers."}), lockFlags); Value * arg = evalState->allocValue(); - evalState->mkAttrs(*arg, 1); + evalState->mkAttrs(*arg, 2); PathSet context; for (auto & i : app.context) context.insert("=" + store->printStorePath(i.path)); mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); + + arg->attrs->sort(); + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1190d7997..80d8654bc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -374,8 +374,9 @@ struct CmdFlakeCheck : FlakeCommand if (v.type != tLambda) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || - v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) - throw Error("bundler must take formal argument 'program'"); + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -505,21 +506,13 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultBundler") - for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkBundler(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); - } + checkBundler(name, vOutput, pos); else if (name == "bundlers") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkBundler( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); - } + for (auto & attr : *vOutput.attrs) + checkBundler(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); } else From 0bdd6cf6ea9bc9779c1767e3cf0207b0373b3b52 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 16:37:44 -0500 Subject: [PATCH 154/207] Add test for builtins.path --- tests/filter-source.sh | 18 ++++++++++++------ tests/path.nix | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 tests/path.nix diff --git a/tests/filter-source.sh b/tests/filter-source.sh index 1f8dceee5..ba34d2eac 100644 --- a/tests/filter-source.sh +++ b/tests/filter-source.sh @@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak touch $TEST_ROOT/filterin/bla.c.bak ln -s xyzzy $TEST_ROOT/filterin/link -nix-build ./filter-source.nix -o $TEST_ROOT/filterout +checkFilter() { + test ! -e $1/foo/bar + test -e $1/xyzzy + test -e $1/bak + test ! -e $1/bla.c.bak + test ! -L $1/link +} -test ! -e $TEST_ROOT/filterout/foo/bar -test -e $TEST_ROOT/filterout/xyzzy -test -e $TEST_ROOT/filterout/bak -test ! -e $TEST_ROOT/filterout/bla.c.bak -test ! -L $TEST_ROOT/filterout/link +nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 +checkFilter $TEST_ROOT/filterout1 + +nix-build ./path.nix -o $TEST_ROOT/filterout2 +checkFilter $TEST_ROOT/filterout2 diff --git a/tests/path.nix b/tests/path.nix new file mode 100644 index 000000000..883c3c41b --- /dev/null +++ b/tests/path.nix @@ -0,0 +1,14 @@ +with import ./config.nix; + +mkDerivation { + name = "filter"; + builder = builtins.toFile "builder" "ln -s $input $out"; + input = + builtins.path { + path = ((builtins.getEnv "TEST_ROOT") + "/filterin"); + filter = path: type: + type != "symlink" + && baseNameOf path != "foo" + && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); + }; +} From cdc23866444b1ef863f917ec0adaaa9bec31126b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 16:40:21 -0500 Subject: [PATCH 155/207] Make expectedHash optional in prim_path This fixes an error found in builtins.path that looks like: store path mismatch in (possibly filtered) path added from '/private/tmp/nix-shell.CyXViH/nix-test/filter-source/filterin' when no hash is specified --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a98cadc80..05d499d1f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1199,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash(htSHA256); + std::optional expectedHash; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); From e3a2154f5ac91a5cbab5d0715984972e1dd7d40d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 31 Jul 2020 01:07:59 +0000 Subject: [PATCH 156/207] Fix indentation --- src/libfetchers/mercurial.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index aee42e136..3e76ffc4d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -209,7 +209,7 @@ struct MercurialInputScheme : InputScheme }); if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); From 5a09eb86f17b7949b93724ecb51f1b909ea68592 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 29 Jun 2020 10:05:44 +0200 Subject: [PATCH 157/207] nix/build: add `--rebuild` option Occasionally, `nix-build --check` is fairly helpful and I'd like to be able to use this feature for flakes that need to be built with `nix build` as well. --- src/nix/build.cc | 9 ++++++++- src/nix/command.hh | 3 ++- src/nix/installables.cc | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 0f7e0e123..bec25d8d8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,6 +9,7 @@ using namespace nix; struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; + BuildMode buildMode = bmNormal; CmdBuild() { @@ -26,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "do not create a symlink to the build result", .handler = {&outLink, Path("")}, }); + + addFlag({ + .longName = "rebuild", + .description = "Rebuild an already built package locally and compare the result to the existing store-paths.", + .handler = {&buildMode, bmCheck}, + }); } std::string description() override @@ -53,7 +60,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile void run(ref store) override { - auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables); + auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); if (dryRun) return; diff --git a/src/nix/command.hh b/src/nix/command.hh index 856721ebf..bc46a2028 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -5,6 +5,7 @@ #include "common-eval-args.hh" #include "path.hh" #include "flake/lockfile.hh" +#include "store-api.hh" #include @@ -185,7 +186,7 @@ static RegisterCommand registerCommand(const std::string & name) } Buildables build(ref store, Realise mode, - std::vector> installables); + std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, Realise mode, OperateOn operateOn, diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a13e5a3df..56c62dbfe 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -642,7 +642,7 @@ std::shared_ptr SourceExprCommand::parseInstallable( } Buildables build(ref store, Realise mode, - std::vector> installables) + std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; @@ -668,7 +668,7 @@ Buildables build(ref store, Realise mode, if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild); + store->buildPaths(pathsToBuild, bMode); return buildables; } From f56dfce7342d66fe81796f039afc283f84ca10fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2020 17:30:12 +0200 Subject: [PATCH 158/207] nix bundle: Set category --- src/nix/bundle.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index f2b78eea5..eb3339f5d 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -48,6 +48,8 @@ struct CmdBundle : InstallableCommand }; } + Category category() override { return catSecondary; } + Strings getDefaultFlakeAttrPaths() override { Strings res{"defaultApp." + settings.thisSystem.get()}; From a3f9625818ecb0c8a3c22c191340dac5a3120bb5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2020 17:32:16 +0200 Subject: [PATCH 159/207] Tweak description --- src/nix/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index bec25d8d8..613cc15eb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -30,7 +30,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile addFlag({ .longName = "rebuild", - .description = "Rebuild an already built package locally and compare the result to the existing store-paths.", + .description = "rebuild an already built package and compare the result to the existing store paths", .handler = {&buildMode, bmCheck}, }); } From 3cbee1e840ea1beff566555f1221b2791091e20c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Aug 2020 15:26:57 +0000 Subject: [PATCH 160/207] Convert to C-style comments --- src/libutil/hash.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index d4badbab3..ffc397ce0 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -42,9 +42,9 @@ struct Hash is not present, then the hash type must be specified in the string. */ static Hash parseAny(std::string_view s, std::optional type); - // hash type must be part of string + /* hash type must be part of string */ static Hash parseAnyPrefixed(std::string_view s); - // prefix parsed separately; non SRI hash + /* prefix parsed separately; non SRI hash */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); From bc165e28aee689a45535afda8012c9b63f87b24c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Aug 2020 15:32:20 +0000 Subject: [PATCH 161/207] Embelish documentation of new Hash functions --- src/libutil/hash.hh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ffc397ce0..00ce7bb6f 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -36,21 +36,26 @@ struct Hash /* Create a zero-filled hash object. */ Hash(HashType type); - /* Initialize the hash from a string representation, in the format + /* Parse the hash from a string representation in the format "[:]" or "-" (a Subresource Integrity hash expression). If the 'type' argument is not present, then the hash type must be specified in the string. */ static Hash parseAny(std::string_view s, std::optional type); - /* hash type must be part of string */ + + /* Parse a hash from a string representation like the above, except the + type prefix is mandatory is there is no separate arguement. */ static Hash parseAnyPrefixed(std::string_view s); - /* prefix parsed separately; non SRI hash */ + + /* Parse a plain hash that musst not have any prefix indicating the type. + The type is passed in to disambiguate. */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); private: - // type must be provided, s must not include prefix + /* The type must be provided, the string view must not include + prefix. `isSRI` helps disambigate the various base-* encodings. */ Hash(std::string_view s, HashType type, bool isSRI); public: From c4ada76e860a595e3f034b89f27374ce79513d9f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Aug 2020 16:22:50 +0000 Subject: [PATCH 162/207] Fix error message and avoid recalculation --- src/libfetchers/fetchers.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 08b83b0db..9c69fc564 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -203,8 +203,8 @@ std::optional Input::getNarHash() const if (auto s = maybeGetStrAttr(attrs, "narHash")) { auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); if (hash.type != htSHA256) - throw UsageError("narHash must be specified with SRI notation"); - return newHashAllowEmpty(*s, htSHA256); + throw UsageError("narHash must use SHA-256"); + return hash; } return {}; } From 54281f3ac130c21407e5ed4326d1d57626d6c19b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Aug 2020 04:13:45 +0000 Subject: [PATCH 163/207] `addToStore` in terms of `addToStoreFromDump` is not local-store-specific --- src/libstore/local-store.cc | 14 -------------- src/libstore/local-store.hh | 10 +--------- src/libstore/store-api.cc | 14 ++++++++++++++ src/libstore/store-api.hh | 6 +++++- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f6ce8877e..f908f7d67 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1038,20 +1038,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); - }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); -} - - StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5a87ec2f8..31e6587ac 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -145,16 +145,8 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; - - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 33f931442..fb9e30597 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -239,6 +239,20 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +StorePath Store::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} + + /* The aim of this function is to compute in one pass the correct ValidPathInfo for the files that we are trying to add to the store. To accomplish that in one diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 49da19496..e05a19975 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -456,7 +456,7 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -465,6 +465,10 @@ public: FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, std::optional expectedCAHash = {}); + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) From 24e07c428f21f28df2a41a7a9851d5867f34753a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2020 18:33:39 +0200 Subject: [PATCH 164/207] Delete compressed NARs Fixes #3891. --- src/libstore/binary-cache-store.cc | 2 ++ src/libutil/util.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 30150eeba..3d3195e03 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto [fdTemp, fnTemp] = createTempFile(); + AutoDelete autoDelete(fnTemp); + auto now1 = std::chrono::steady_clock::now(); /* Read the NAR simultaneously into a CompressionSink+FileSink (to diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 97d278581..8bc60ec2d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -494,6 +494,7 @@ std::pair createTempFile(const Path & prefix) { Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); From fe7e57a80d1f698565aaa13596de3fa13dbafe1f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 03:46:28 +0000 Subject: [PATCH 165/207] tab -> space --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index babb0e5fe..59b52ce95 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -437,7 +437,7 @@ ref openEvalCache( std::shared_ptr lockedFlake, bool useEvalCache) { - auto fingerprint = lockedFlake->getFingerprint(); + auto fingerprint = lockedFlake->getFingerprint(); return make_ref( useEvalCache && evalSettings.pureEval ? std::optional { std::cref(fingerprint) } From 327b1bf378ebdfe7acc6357b651470b45cb44472 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2020 14:50:43 +0200 Subject: [PATCH 166/207] BinaryCacheStore: Explicitly flush file sink The file sink is also flushed in its destructor, but we ignore any exceptions in the destructor. Issue #3886. --- src/libstore/binary-cache-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3d3195e03..7d103e0cc 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -169,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource TeeSource teeSource(narSource, *compressionSink); narAccessor = makeNarAccessor(teeSource); compressionSink->finish(); + fileSink.flush(); } auto now2 = std::chrono::steady_clock::now(); From dfe66420e7e5395acfa8068fafcadc3a38a56184 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2020 15:56:10 +0200 Subject: [PATCH 167/207] Revert "Remove putBytes" This reverts commit b8eea7e81af53905be7845dffc6d0a83ea8edc97. --- src/libstore/s3-binary-cache-store.cc | 7 +++++-- src/libstore/s3-binary-cache-store.hh | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 67935f3ba..1b7dff085 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -343,10 +343,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - printInfo("uploaded 's3://%s/%s' in %d ms", - bucketName, path, duration); + auto size = istream->tellg(); + + printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, size, duration); stats.putTimeMs += duration; + stats.putBytes += size; stats.put++; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index b2b75d498..4d43fe4d2 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -19,6 +19,7 @@ public: struct Stats { std::atomic put{0}; + std::atomic putBytes{0}; std::atomic putTimeMs{0}; std::atomic get{0}; std::atomic getBytes{0}; From 4e7f1c7f11fb22d7954b2dacbe87c61548fab82d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2020 16:00:59 +0200 Subject: [PATCH 168/207] S3BinaryCacheStore: Fix size determination --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7d103e0cc..9682db730 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -283,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, - std::make_shared(fnTemp, std::ios_base::in), + std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary), "application/x-nix-nar"); } else stats.narWriteAverted++; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085..a0a446bd3 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const std::string & mimeType, const std::string & contentEncoding) { + istream->seekg(0, istream->end); + auto size = istream->tellg(); + istream->seekg(0, istream->beg); + auto maxThreads = std::thread::hardware_concurrency(); static std::shared_ptr @@ -343,13 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - auto size = istream->tellg(); - printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, size, duration); stats.putTimeMs += duration; - stats.putBytes += size; + stats.putBytes += std::max(size, (decltype(size)) 0); stats.put++; } From 6d9ccde20d783a660aca3cab0e6e5c20e8444641 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 01:09:52 +0000 Subject: [PATCH 169/207] Make JSON equality tests agnostic to ordering It is in fact more sorted than before, but I don't think we want to guarantee anything about the ordering. --- tests/binary-cache.sh | 8 ++++++-- tests/nar-access.sh | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 40f1a4f76..b05a7d167 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -218,7 +218,9 @@ outPath=$(nix-build --no-out-link -E ' nix copy --to file://$cacheDir?write-nar-listing=1 $outPath -[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]] +diff -u \ + <(jq -S < $cacheDir/$(basename $outPath).ls) \ + <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) # Test debug info index generation. @@ -234,4 +236,6 @@ outPath=$(nix-build --no-out-link -E ' nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath -[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]] +diff -u \ + <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ + <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) diff --git a/tests/nar-access.sh b/tests/nar-access.sh index 553d6ca89..88b997ca6 100644 --- a/tests/nar-access.sh +++ b/tests/nar-access.sh @@ -26,12 +26,24 @@ nix cat-store $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz # Test --json. -[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] -[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]] -[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]] -[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] -[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]] -[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]] +diff -u \ + <(nix ls-nar --json $narFile / | jq -S) \ + <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) +diff -u \ + <(nix ls-nar --json -R $narFile /foo | jq -S) \ + <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) +diff -u \ + <(nix ls-nar --json -R $narFile /foo/bar | jq -S) \ + <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) +diff -u \ + <(nix ls-store --json $storePath | jq -S) \ + <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) +diff -u \ + <(nix ls-store --json -R $storePath/foo | jq -S) \ + <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) +diff -u \ + <(nix ls-store --json -R $storePath/foo/bar| jq -S) \ + <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' From 088dcea0e80bf2861fd9d6b808e76a1669b7122a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2020 15:41:51 +0200 Subject: [PATCH 170/207] Typo --- src/libexpr/eval.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a2f55504..ecac5d522 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1256,10 +1256,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po try { lambda.body->eval(*this, env2, v); } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambdaction")); + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); addErrorTrace(e, pos, "from call site%s", ""); throw; } From e7b0847f2d9674bc18532c86b2daf421347513e4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 14:44:39 +0000 Subject: [PATCH 171/207] Make names more consistent --- src/libexpr/primops.cc | 2 +- src/libstore/build.cc | 10 +++++----- src/libstore/derivations.cc | 36 ++++++++++++++++++------------------ src/libstore/derivations.hh | 14 ++++++++++---- src/libstore/local-store.cc | 4 ++-- src/libstore/misc.cc | 2 +- src/nix/show-derivation.cc | 4 ++-- 7 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a9b5a10c9..6fec028be 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -774,7 +774,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = ingestionMethod, .hash = std::move(h), diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0176e7f65..6c9f55a53 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3722,18 +3722,18 @@ void DerivationGoal::registerOutputs() std::optional ca; if (! std::holds_alternative(i.second.output)) { - DerivationOutputFloating outputHash; + DerivationOutputCAFloating outputHash; std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { assert(false); // Enclosing `if` handles this case in other branch }, - [&](DerivationOutputFixed dof) { - outputHash = DerivationOutputFloating { + [&](DerivationOutputCAFixed dof) { + outputHash = DerivationOutputCAFloating { .method = dof.hash.method, .hashType = dof.hash.hash.type, }; }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { outputHash = dof; }, }, i.second.output); @@ -3758,7 +3758,7 @@ void DerivationGoal::registerOutputs() // true if either floating CA, or incorrect fixed hash. bool needsMove = true; - if (auto p = std::get_if(& i.second.output)) { + if (auto p = std::get_if(& i.second.output)) { Hash & h = p->hash.hash; if (h != h2) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8f2339885..b17d0cf79 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -13,12 +13,12 @@ std::optional DerivationOutput::pathOpt(const Store & store, std::str [](DerivationOutputInputAddressed doi) -> std::optional { return { doi.path }; }, - [&](DerivationOutputFixed dof) -> std::optional { + [&](DerivationOutputCAFixed dof) -> std::optional { return { store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName) }; }, - [](DerivationOutputFloating dof) -> std::optional { + [](DerivationOutputCAFloating dof) -> std::optional { return std::nullopt; }, }, output); @@ -27,7 +27,7 @@ std::optional DerivationOutput::pathOpt(const Store & store, std::str bool derivationIsCA(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return true; }; @@ -38,7 +38,7 @@ bool derivationIsCA(DerivationType dt) { bool derivationIsFixed(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; }; @@ -47,7 +47,7 @@ bool derivationIsFixed(DerivationType dt) { bool derivationIsImpure(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; }; @@ -156,7 +156,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings return hash != "" ? DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = std::move(method), .hash = Hash::parseNonSRIUnprefixed(hash, hashType), @@ -164,7 +164,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings } } : DerivationOutput { - .output = DerivationOutputFloating { + .output = DerivationOutputCAFloating { .method = std::move(method), .hashType = std::move(hashType), }, @@ -321,11 +321,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, @@ -390,10 +390,10 @@ DerivationType BasicDerivation::type() const [&](DerivationOutputInputAddressed _) { inputAddressedOutputs.insert(i.first); }, - [&](DerivationOutputFixed _) { + [&](DerivationOutputCAFixed _) { fixedCAOutputs.insert(i.first); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { floatingCAOutputs.insert(i.first); if (!floatingHashType) { floatingHashType = dof.hashType; @@ -408,7 +408,7 @@ DerivationType BasicDerivation::type() const if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { throw Error("Must have at least one output"); } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { - return DerivationType::Regular; + return DerivationType::InputAddressed; } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? @@ -474,7 +474,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.output); + auto & dof = std::get(i.second.output); auto hash = hashString(htSHA256, "fixed:out:" + dof.hash.printMethodAlgo() + ":" + dof.hash.hash.to_string(Base16, false) + ":" @@ -483,7 +483,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } - case DerivationType::Regular: + case DerivationType::InputAddressed: break; } @@ -552,7 +552,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashType = parseHashType(hashAlgo); return hash != "" ? DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = std::move(method), .hash = Hash::parseNonSRIUnprefixed(hash, hashType), @@ -560,7 +560,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) } } : DerivationOutput { - .output = DerivationOutputFloating { + .output = DerivationOutputCAFloating { .method = std::move(method), .hashType = std::move(hashType), }, @@ -628,11 +628,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](DerivationOutputInputAddressed doi) { out << "" << ""; }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { out << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1cda85cb..5a410a164 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -14,6 +14,7 @@ namespace nix { /* Abstract syntax of derivations. */ +/* The traditional non-fixed-output derivation type. */ struct DerivationOutputInputAddressed { /* Will need to become `std::optional` once input-addressed @@ -21,12 +22,17 @@ struct DerivationOutputInputAddressed StorePath path; }; -struct DerivationOutputFixed +/* Fixed-output derivations, whose output paths are content addressed + according to that fixed output. */ +struct DerivationOutputCAFixed { FixedOutputHash hash; /* hash used for expected hash computation */ }; -struct DerivationOutputFloating +/* Floating-output derivations, whose output paths are content addressed, but + not fixed, and so are dynamically calculated from whatever the output ends + up being. */ +struct DerivationOutputCAFloating { /* information used for expected hash computation */ FileIngestionMethod method; @@ -37,8 +43,8 @@ struct DerivationOutput { std::variant< DerivationOutputInputAddressed, - DerivationOutputFixed, - DerivationOutputFloating + DerivationOutputCAFixed, + DerivationOutputCAFloating > output; std::optional hashAlgoOpt(const Store & store) const; std::optional pathOpt(const Store & store, std::string_view drvName) const; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7de065ba8..de7ddb84b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -573,11 +573,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); envHasRightPath(doia.path, i.first); }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); envHasRightPath(path, i.first); }, - [&](DerivationOutputFloating _) { + [&](DerivationOutputCAFloating _) { throw UnimplementedError("Floating CA output derivations are not yet implemented"); }, }, i.second.output); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ddba5d052..0ae1ceaad 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -113,7 +113,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end()) { - if (auto v = std::get_if(&out->second.output)) + if (auto v = std::get_if(&out->second.output)) return v->hash; } return std::nullopt; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 25ea19834..1b51d114f 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -74,11 +74,11 @@ struct CmdShowDerivation : InstallablesCommand std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, }, output.second.output); From e561a13a5863f25c81e8abc9d235a12925fd454e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 14:45:56 +0000 Subject: [PATCH 172/207] Reanme `DerivationType::Regular` defintion too This is the one non-prefixed occurence --- src/libstore/derivations.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 5a410a164..14e0e947a 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -65,7 +65,7 @@ typedef std::map DerivationInputs; typedef std::map StringPairs; enum struct DerivationType : uint8_t { - Regular, + InputAddressed, CAFixed, CAFloating, }; From 25f79121564b21ec7c84f33c9348b169f20d2bdc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2020 16:47:48 +0200 Subject: [PATCH 173/207] Style fix --- src/libstore/content-address.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 749551d1a..6428aa736 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -48,14 +48,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); + throw UsageError("not a content address because it is not in the form ':': %s", rawCa); prefix = *optPrefix; } auto parseHashType_ = [&](){ auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) - throw UsageError("content address hash must be in form \":\", but found: %s", rawCa); + throw UsageError("content address hash must be in form ':', but found: %s", rawCa); HashType hashType = parseHashType(*hashTypeRaw); return std::move(hashType); }; @@ -81,7 +81,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else - throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); + throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { From b9ebe373bbab6f19ee650ef9769ad76c32b7244d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 14:49:25 +0000 Subject: [PATCH 174/207] Sed some names to perhaps avoid conflicts --- src/libexpr/primops.cc | 4 ++-- src/libstore/build.cc | 10 ++++----- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/derivations.cc | 36 +++++++++++++++---------------- src/libstore/derivations.hh | 10 ++++----- src/libstore/local-store.cc | 4 ++-- src/nix/show-derivation.cc | 4 ++-- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 784c12b16..7bc424d52 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -783,7 +783,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = ingestionMethod, .hash = std::move(h), @@ -797,7 +797,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { if (!jsonObject) drv.env[i] = hashPlaceholder(i); drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputFloating { + .output = DerivationOutputCAFloating { .method = ingestionMethod, .hashType = std::move(ht), }, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7c8323c29..38897a9e2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3722,18 +3722,18 @@ void DerivationGoal::registerOutputs() std::optional ca; if (! std::holds_alternative(i.second.output)) { - DerivationOutputFloating outputHash; + DerivationOutputCAFloating outputHash; std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { assert(false); // Enclosing `if` handles this case in other branch }, - [&](DerivationOutputFixed dof) { - outputHash = DerivationOutputFloating { + [&](DerivationOutputCAFixed dof) { + outputHash = DerivationOutputCAFloating { .method = dof.hash.method, .hashType = dof.hash.hash.type, }; }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { outputHash = dof; }, }, i.second.output); @@ -3758,7 +3758,7 @@ void DerivationGoal::registerOutputs() // true if either floating CA, or incorrect fixed hash. bool needsMove = true; - if (auto p = std::get_if(& i.second.output)) { + if (auto p = std::get_if(& i.second.output)) { Hash & h = p->hash.hash; if (h != h2) { diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 03bb77488..8291c745c 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,7 +63,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto & output = drv.outputs.begin()->second; /* Try the hashed mirrors first. */ - if (auto hash = std::get_if(&output.output)) { + if (auto hash = std::get_if(&output.output)) { if (hash->hash.method == FileIngestionMethod::Flat) { for (auto hashedMirror : settings.hashedMirrors.get()) { try { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6a12e8734..03cdc1760 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -13,12 +13,12 @@ std::optional DerivationOutput::pathOpt(const Store & store, std::str [](DerivationOutputInputAddressed doi) -> std::optional { return { doi.path }; }, - [&](DerivationOutputFixed dof) -> std::optional { + [&](DerivationOutputCAFixed dof) -> std::optional { return { store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName) }; }, - [](DerivationOutputFloating dof) -> std::optional { + [](DerivationOutputCAFloating dof) -> std::optional { return std::nullopt; }, }, output); @@ -27,7 +27,7 @@ std::optional DerivationOutput::pathOpt(const Store & store, std::str bool derivationIsCA(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return true; }; @@ -38,7 +38,7 @@ bool derivationIsCA(DerivationType dt) { bool derivationIsFixed(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; }; @@ -47,7 +47,7 @@ bool derivationIsFixed(DerivationType dt) { bool derivationIsImpure(DerivationType dt) { switch (dt) { - case DerivationType::Regular: return false; + case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; }; @@ -156,7 +156,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings return hash != "" ? DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = std::move(method), .hash = Hash(hash, hashType), @@ -165,7 +165,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings } : (settings.requireExperimentalFeature("ca-derivations"), DerivationOutput { - .output = DerivationOutputFloating { + .output = DerivationOutputCAFloating { .method = std::move(method), .hashType = std::move(hashType), }, @@ -322,11 +322,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, @@ -391,10 +391,10 @@ DerivationType BasicDerivation::type() const [&](DerivationOutputInputAddressed _) { inputAddressedOutputs.insert(i.first); }, - [&](DerivationOutputFixed _) { + [&](DerivationOutputCAFixed _) { fixedCAOutputs.insert(i.first); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { floatingCAOutputs.insert(i.first); if (!floatingHashType) { floatingHashType = dof.hashType; @@ -409,7 +409,7 @@ DerivationType BasicDerivation::type() const if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { throw Error("Must have at least one output"); } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { - return DerivationType::Regular; + return DerivationType::InputAddressed; } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? @@ -475,7 +475,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.output); + auto & dof = std::get(i.second.output); auto hash = hashString(htSHA256, "fixed:out:" + dof.hash.printMethodAlgo() + ":" + dof.hash.hash.to_string(Base16, false) + ":" @@ -484,7 +484,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } - case DerivationType::Regular: + case DerivationType::InputAddressed: break; } @@ -553,7 +553,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashType = parseHashType(hashAlgo); return hash != "" ? DerivationOutput { - .output = DerivationOutputFixed { + .output = DerivationOutputCAFixed { .hash = FixedOutputHash { .method = std::move(method), .hash = Hash(hash, hashType), @@ -562,7 +562,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) } : (settings.requireExperimentalFeature("ca-derivations"), DerivationOutput { - .output = DerivationOutputFloating { + .output = DerivationOutputCAFloating { .method = std::move(method), .hashType = std::move(hashType), }, @@ -630,11 +630,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](DerivationOutputInputAddressed doi) { out << "" << ""; }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { out << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1cda85cb..09d51649e 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -21,12 +21,12 @@ struct DerivationOutputInputAddressed StorePath path; }; -struct DerivationOutputFixed +struct DerivationOutputCAFixed { FixedOutputHash hash; /* hash used for expected hash computation */ }; -struct DerivationOutputFloating +struct DerivationOutputCAFloating { /* information used for expected hash computation */ FileIngestionMethod method; @@ -37,8 +37,8 @@ struct DerivationOutput { std::variant< DerivationOutputInputAddressed, - DerivationOutputFixed, - DerivationOutputFloating + DerivationOutputCAFixed, + DerivationOutputCAFloating > output; std::optional hashAlgoOpt(const Store & store) const; std::optional pathOpt(const Store & store, std::string_view drvName) const; @@ -59,7 +59,7 @@ typedef std::map DerivationInputs; typedef std::map StringPairs; enum struct DerivationType : uint8_t { - Regular, + InputAddressed, CAFixed, CAFloating, }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ccf2a9a27..edcb17607 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -573,11 +573,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); envHasRightPath(doia.path, i.first); }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); envHasRightPath(path, i.first); }, - [&](DerivationOutputFloating _) { + [&](DerivationOutputCAFloating _) { throw UnimplementedError("Floating CA output derivations are not yet implemented"); }, }, i.second.output); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 25ea19834..1b51d114f 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -74,11 +74,11 @@ struct CmdShowDerivation : InstallablesCommand std::visit(overloaded { [&](DerivationOutputInputAddressed doi) { }, - [&](DerivationOutputFixed dof) { + [&](DerivationOutputCAFixed dof) { outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, - [&](DerivationOutputFloating dof) { + [&](DerivationOutputCAFloating dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, }, output.second.output); From 790b694be71c102ca9c67a85a3e9fd6f076811b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2020 16:51:06 +0200 Subject: [PATCH 175/207] Style fix --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index de7ddb84b..3c66a4dfd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -578,7 +578,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { - throw UnimplementedError("Floating CA output derivations are not yet implemented"); + throw UnimplementedError("floating CA output derivations are not yet implemented"); }, }, i.second.output); } From b3e73547a03f068ae4dd9cca4bc865cde85c8dec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 11:05:46 -0400 Subject: [PATCH 176/207] Update src/libexpr/primops.cc Co-authored-by: Eelco Dolstra --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9ecf99f17..65d36ca0e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -647,7 +647,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (i->name == state.sArgs) { + else if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); From a9bbfaa8515de7107457fda2a7aef87aa198788e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 16:27:15 +0000 Subject: [PATCH 177/207] Fix --profile with multiple opaque paths --- src/nix/command.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 04715b43a..da32819da 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -128,27 +128,25 @@ void MixProfile::updateProfile(const Buildables & buildables) { if (!profile) return; - std::optional result; + std::vector result; for (auto & buildable : buildables) { std::visit(overloaded { [&](BuildableOpaque bo) { - result = bo.path; + result.push_back(bo.path); }, [&](BuildableFromDrv bfd) { for (auto & output : bfd.outputs) { - if (result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; + result.push_back(output.second); } }, }, buildable); } - if (!result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + if (result.size() != 1) + throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(*result); + updateProfile(result[0]); } MixDefaultProfile::MixDefaultProfile() From f1a47a96b6ab55d0b0017df5b94b3da69c65bf21 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 5 Aug 2020 10:58:00 -0600 Subject: [PATCH 178/207] error messages for issue 2238 --- src/build-remote/build-remote.cc | 33 +++++++++++++++++++++++++++++++- src/libstore/build.cc | 13 +++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 3579d8fff..2414a47c1 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -103,7 +103,7 @@ static int _main(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + auto canBuildLocally = amWilling && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(requiredFeatures); @@ -170,7 +170,38 @@ static int _main(int argc, char * * argv) if (rightType && !canBuildLocally) std::cerr << "# postpone\n"; else + { + // build the hint template. + string hintstring = "required (system, features): (%s, %s)"; + hintstring += "\n%s available machines:"; + hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; + + for (unsigned int i = 0; i < machines.size(); ++i) { + hintstring += "\n(%s, %s, %s, %s)"; + } + + // add the template values. + auto hint = hintformat(hintstring); + hint + % neededSystem + % concatStringsSep(", ", requiredFeatures) + % machines.size(); + + for (auto & m : machines) { + hint % concatStringsSep>(", ", m.systemTypes) + % m.maxJobs + % concatStringsSep(", ", m.supportedFeatures) + % concatStringsSep(", ", m.mandatoryFeatures); + } + + logError({ + .name = "Remote build", + .description = "Failed to find a machine for remote build!", + .hint = hint + }); + std::cerr << "# decline\n"; + } break; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 94c398b2f..76baa1a6e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4876,8 +4876,17 @@ void Worker::run(const Goals & _topGoals) waitForInput(); else { if (awake.empty() && 0 == settings.maxBuildJobs) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + { + if (getMachines().empty()) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + else + throw Error("unable to start any build; remote machines may not have " + "all required system features." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + + } assert(!awake.empty()); } } From e4eae078a52fa4209bb5a46d21dbed09dd9c54ec Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 5 Aug 2020 11:21:36 -0600 Subject: [PATCH 179/207] add derivation path to hint --- src/build-remote/build-remote.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2414a47c1..723e1463e 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -172,7 +172,7 @@ static int _main(int argc, char * * argv) else { // build the hint template. - string hintstring = "required (system, features): (%s, %s)"; + string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)"; hintstring += "\n%s available machines:"; hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; @@ -183,6 +183,7 @@ static int _main(int argc, char * * argv) // add the template values. auto hint = hintformat(hintstring); hint + % drvPath->to_string() % neededSystem % concatStringsSep(", ", requiredFeatures) % machines.size(); From 31f1af0cabb65e3c5f8f4539500c6b236c387780 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 5 Aug 2020 11:26:06 -0600 Subject: [PATCH 180/207] don't crash if there's no drvPath --- src/build-remote/build-remote.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 723e1463e..5247cefa6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -181,9 +181,15 @@ static int _main(int argc, char * * argv) } // add the template values. + string drvstr; + if (drvPath.has_value()) + drvstr = drvPath->to_string(); + else + drvstr = ""; + auto hint = hintformat(hintstring); hint - % drvPath->to_string() + % drvstr % neededSystem % concatStringsSep(", ", requiredFeatures) % machines.size(); From 1d71028f4d43f8e3ade559f7187f697c7bb9d709 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 5 Aug 2020 14:42:48 -0400 Subject: [PATCH 181/207] Remove optionality in ValidPathInfo::narInfo --- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 4 ++-- src/libstore/build.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/export-import.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 14 +++++++------- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 6 +++--- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- src/nix-store/nix-store.cc | 10 +++++----- src/nix/make-content-addressable.cc | 2 +- src/nix/profile.cc | 2 +- src/nix/verify.cc | 8 ++++---- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cddcf0e59..0dbf4ae1d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9c69fc564..eaa635595 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -130,12 +130,12 @@ std::pair Input::fetch(ref store) const tree.actualPath = store->toRealPath(tree.storePath); auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true)); + to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); } if (auto prevLastModified = getLastModified()) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4156cd678..51dd0c094 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -5037,7 +5037,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5e568fc94..fa3bf6e2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -289,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash->to_string(Base16, false); + to << hash.to_string(Base16, false); break; } @@ -638,7 +638,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash->to_string(Base16, false); + << info->narHash.to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index a0fc22264..3f4b7c36f 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -38,9 +38,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); teeSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c6eeab548..ed57a188b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? std::optional{} : Hash::parseAnyPrefixed(s); + info->narHash = s.empty() ? Hash(htSHA256) : Hash::parseAnyPrefixed(s); info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -138,7 +138,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash->to_string(Base16, false); + << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6b0548538..93a86f828 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -579,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash->to_string(Base16, true)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -679,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash->to_string(Base16, true)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -905,7 +905,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash && i.narHash->type == htSHA256); + assert(i.narHash && i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1018,7 +1018,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1300,9 +1300,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr hashSink; if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(info->narHash->type); + hashSink = std::make_unique(info->narHash.type); else - hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1311,7 +1311,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 92da14e23..dd40006d7 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -232,7 +232,7 @@ public: (narInfo ? narInfo->compression : "", narInfo != 0) (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash->to_string(Base32, true)) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 5812aa4ac..cadaf6267 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -89,8 +89,8 @@ std::string NarInfo::to_string(const Store & store) const assert(fileHash && fileHash->type == htSHA256); res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash && narHash->type == htSHA256); - res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; + assert(narHash && narHash.type == htSHA256); + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 2a015ea3c..ba154dab7 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,7 +29,7 @@ struct ValidPathInfo StorePath path; std::optional deriver; // TODO document this - std::optional narHash; + Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -100,8 +100,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path) : path(std::move(path)) { }; - ValidPathInfo(const StorePath & path) : path(path) { }; + ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash(htSHA256)) { }; + ValidPathInfo(const StorePath & path) : path(path), narHash(Hash(htSHA256)) { }; virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 33d1e431b..b5e4d9fa6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -521,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash->to_string(Base16, false); + << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ea666f098..81180c484 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -569,7 +569,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash->to_string(Base16, false) + "\n"; + s += info->narHash.to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -601,7 +601,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash->to_string(hashBase, true)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -919,7 +919,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash->to_string(Base32, true) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 2996e36c4..a901261f4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash && info->narHash->type == htSHA256); - cout << fmt("%s\n", info->narHash->to_string(Base32, true)); + assert(info->narHash && info->narHash.type == htSHA256); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -723,7 +723,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash->type); + HashSink sink(info->narHash.type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -732,7 +732,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash->to_string(Base32, true), + info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -862,7 +862,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash->to_string(Base32, true) : "") + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 2fe2e2fb2..712043978 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -84,7 +84,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, - .hash = *info.narHash, + .hash = info.narHash, }; if (!json) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7dcc0b6d4..c6cd88c49 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -133,7 +133,7 @@ struct ProfileManifest info.references = std::move(references); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash }; + info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index fc7a9765c..26f755fd9 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -91,15 +91,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (!info->ca) - hashSink = std::make_unique(info->narHash->type); + hashSink = std::make_unique(info->narHash.type); else - hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != *info->narHash) { + if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -107,7 +107,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash->to_string(Base32, true), + info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true)) }); } From 1ad6394b33b5e627c27bc26247f8a5e1d7d81ce1 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 5 Aug 2020 15:11:49 -0400 Subject: [PATCH 182/207] Add Hash::dummy to signal default value We did this in the same spirit of the dummy value that's present in libstore/path.hh --- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/path-info.hh | 4 ++-- src/libutil/hash.cc | 2 ++ src/libutil/hash.hh | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ed57a188b..f0f5ec626 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash(htSHA256) : Hash::parseAnyPrefixed(s); + info->narHash = s.empty() ? Hash::dummy : Hash::parseAnyPrefixed(s); info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index ba154dab7..15484717e 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -100,8 +100,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash(htSHA256)) { }; - ValidPathInfo(const StorePath & path) : path(path), narHash(Hash(htSHA256)) { }; + ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash::dummy) { }; + ValidPathInfo(const StorePath & path) : path(path), narHash(Hash::dummy) { }; virtual ~ValidPathInfo() { } }; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index dfb3668f1..4a94f0dfd 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -136,6 +136,8 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } +Hash Hash::dummy(htSHA256); + Hash Hash::parseSRI(std::string_view original) { auto rest = original; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 00ce7bb6f..0520c6022 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -105,6 +105,8 @@ public: assert(type == htSHA1); return std::string(to_string(Base16, false), 0, 7); } + + static Hash dummy; }; /* Helper that defaults empty hashes to the 0 hash. */ From 8241e660bac5583d2924a490e5b32027e9e8e2fc Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 5 Aug 2020 15:30:38 -0400 Subject: [PATCH 183/207] Remove Hash::operator bool () Since the hash is not optional anymore --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 5 +---- src/libstore/nar-info.cc | 4 ++-- src/libstore/store-api.cc | 18 ++---------------- src/libutil/hash.hh | 3 --- src/nix-store/nix-store.cc | 4 ++-- 6 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9682db730..4bcb2c93b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -143,7 +143,7 @@ struct FileSource : FdSource void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) { - assert(info.narHash && info.narSize); + assert(info.narSize); if (!repair && isValidPath(info.path)) { // FIXME: copyNAR -> null sink diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3542904bf..fc0630cae 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -920,7 +920,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash && i.narHash.type == htSHA256); + assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -984,9 +984,6 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index cadaf6267..3a9121679 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -76,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); + if (!havePath || url.empty() || narSize == 0) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -89,7 +89,7 @@ std::string NarInfo::to_string(const Store & store) const assert(fileHash && fileHash->type == htSHA256); res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash && narHash.type == htSHA256); + assert(narHash.type == htSHA256); res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index aead3468a..997af1195 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -725,20 +725,6 @@ void copyStorePath(ref srcStore, ref dstStore, info = info2; } - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } - if (info->ultimate) { auto info2 = make_ref(*info); info2->ultimate = false; @@ -910,8 +896,8 @@ string showPaths(const PathSet & paths) std::string ValidPathInfo::fingerprint(const Store & store) const { - if (narSize == 0 || !narHash) - throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0520c6022..6d6eb70ca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -59,9 +59,6 @@ private: Hash(std::string_view s, HashType type, bool isSRI); public: - /* Check whether a hash is set. */ - operator bool () const { return (bool) type; } - /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a901261f4..cc104bf35 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash && info->narHash.type == htSHA256); + assert(info->narHash.type == htSHA256); cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); @@ -862,7 +862,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") + out << info->narHash.to_string(Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { From d5d9907a9ebd89e84d5031cd767dd4d97145803a Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 5 Aug 2020 15:57:42 -0400 Subject: [PATCH 184/207] Fix perl integration --- perl/lib/Nix/Store.xs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de60fb939..41846e6a7 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash->to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); From 59067f0f587f75908c4f990bcbab29340c7f7652 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Aug 2020 11:40:41 +0200 Subject: [PATCH 185/207] repl.cc: Check for HAVE_BOEHMGC Fixes #3906. --- src/nix/repl.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fb9050d0d..c3c9e54a8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -33,12 +33,17 @@ extern "C" { #include "command.hh" #include "finally.hh" +#if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include +#endif namespace nix { -struct NixRepl : gc +struct NixRepl + #if HAVE_BOEHMGC + : gc + #endif { string curDir; std::unique_ptr state; From e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Aug 2020 18:31:48 +0000 Subject: [PATCH 186/207] Minimize the usage of `Hash::dummy` --- src/libfetchers/tarball.cc | 6 ++++-- src/libstore/binary-cache-store.cc | 10 ++++++++-- src/libstore/build.cc | 6 ++++-- src/libstore/daemon.cc | 5 +++-- src/libstore/export-import.cc | 11 ++++++----- src/libstore/legacy-ssh-store.cc | 19 +++++++++++++------ src/libstore/local-store.cc | 21 +++++++++++---------- src/libstore/nar-info-disk-cache.cc | 5 +++-- src/libstore/nar-info.cc | 10 +++++++--- src/libstore/nar-info.hh | 6 ++++-- src/libstore/path-info.hh | 5 +++-- src/libstore/remote-store.cc | 4 ++-- src/libstore/store-api.cc | 19 ++++++++++++------- src/libstore/store-api.hh | 3 +-- src/nix-store/nix-store.cc | 11 ++++++++--- src/nix/add-to-store.cc | 6 ++++-- src/nix/make-content-addressable.cc | 6 ++++-- src/nix/profile.cc | 6 ++++-- 18 files changed, 101 insertions(+), 58 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55158cece..a2d16365e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,8 +67,10 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + hashString(htSHA256, *sink.s), + }; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Flat, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4bcb2c93b..9608467e2 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -385,7 +385,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, *h, name)); + ValidPathInfo info { + makeFixedOutputPath(method, *h, name), + Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash + }; auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs); @@ -396,7 +399,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info(computeStorePathForText(name, s, references)); + ValidPathInfo info { + computeStorePathForText(name, s, references), + Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash + }; info.references = references; if (repair || !isValidPath(info.path)) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3370af63d..9effd7ec0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3864,8 +3864,10 @@ void DerivationGoal::registerOutputs() worker.markContentsGood(worker.store.parseStorePath(path)); } - ValidPathInfo info(worker.store.parseStorePath(path)); - info.narHash = hash.first; + ValidPathInfo info { + worker.store.parseStorePath(path), + hash.first, + }; info.narSize = hash.second; info.references = std::move(references); info.deriver = drvPath; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fa3bf6e2d..7f7f25ebf 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -694,11 +694,12 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddToStoreNar: { bool repair, dontCheckSigs; - ValidPathInfo info(store->parseStorePath(readString(from))); + auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); + auto narHash = Hash::parseAny(readString(from), htSHA256); + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash::parseAny(readString(from), htSHA256); info.references = readStorePaths(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 3f4b7c36f..ccd466d09 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -69,17 +69,18 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info(parseStorePath(readString(source))); + auto path = parseStorePath(readString(source)); //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); - + auto references = readStorePaths(*this, source); auto deriver = readString(source); + auto narHash = hashString(htSHA256, *saved.s); + + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - - info.narHash = hashString(htSHA256, *saved.s); + info.references = references; info.narSize = saved.s->size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index f0f5ec626..9f95a9726 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -93,6 +93,9 @@ struct LegacySSHStore : public Store try { auto conn(connections->get()); + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; @@ -100,8 +103,10 @@ struct LegacySSHStore : public Store auto p = readString(conn->from); if (p.empty()) return callback(nullptr); - auto info = std::make_shared(parseStorePath(p)); - assert(path == info->path); + auto path2 = parseStorePath(p); + assert(path == path2); + /* Hash will be set below. FIXME construct ValidPathInfo at end. */ + auto info = std::make_shared(path, Hash::dummy); PathSet references; auto deriver = readString(conn->from); @@ -111,12 +116,14 @@ struct LegacySSHStore : public Store readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash::dummy : Hash::parseAnyPrefixed(s); - info->ca = parseContentAddressOpt(readString(conn->from)); - info->sigs = readStrings(conn->from); + if (s == "") + throw Error("NAR hash is now mandatory"); + info->narHash = Hash::parseAnyPrefixed(s); } + info->ca = parseContentAddressOpt(readString(conn->from)); + info->sigs = readStrings(conn->from); auto s = readString(conn->from); assert(s == ""); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fc0630cae..c96db68ed 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -641,25 +641,28 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - auto info = std::make_shared(path); - callback(retrySQLite>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr(); - info->id = useQueryPathInfo.getInt(0); + auto id = useQueryPathInfo.getInt(0); + auto narHash = Hash::dummy; try { - info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); } + auto info = std::make_shared(path, narHash); + + info->id = id; + info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); @@ -1152,8 +1155,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = narHash.first; + ValidPathInfo info { dstPath, narHash.first }; info.narSize = narHash.second; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); @@ -1196,8 +1198,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = narHash; + ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s->size(); info.references = references; info.ca = TextHash { .hash = hash }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index dd40006d7..8541cc51f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,13 +189,14 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref(StorePath(hashPart + "-" + namePart)); + auto narInfo = make_ref( + StorePath(hashPart + "-" + namePart), + Hash::parseAnyPrefixed(queryNAR.getStr(6))); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4)); narInfo->fileSize = queryNAR.getInt(5); - narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) narInfo->references.insert(StorePath(r)); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3a9121679..3454f34bb 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,10 +1,11 @@ #include "globals.hh" #include "nar-info.hh" +#include "store-api.hh" namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { auto corrupt = [&]() { return Error("NAR info file '%1%' is corrupt", whence); @@ -19,6 +20,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & }; bool havePath = false; + bool haveNarHash = false; size_t pos = 0; while (pos < s.size()) { @@ -46,8 +48,10 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileSize") { if (!string2Int(value, fileSize)) throw corrupt(); } - else if (name == "NarHash") + else if (name == "NarHash") { narHash = parseHashField(value); + haveNarHash = true; + } else if (name == "NarSize") { if (!string2Int(value, narSize)) throw corrupt(); } @@ -76,7 +80,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index eff19f0ef..39ced76e5 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -2,10 +2,12 @@ #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "path-info.hh" namespace nix { +class Store; + struct NarInfo : ValidPathInfo { std::string url; @@ -15,7 +17,7 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; - NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } + NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 15484717e..8ff5c466e 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,5 +1,6 @@ #pragma once +#include "crypto.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -100,8 +101,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash::dummy) { }; - ValidPathInfo(const StorePath & path) : path(path), narHash(Hash::dummy) { }; + ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; + ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b5e4d9fa6..8dcc1d710 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -419,10 +419,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared(StorePath(path)); auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + info = std::make_shared(path, narHash); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash::parseAny(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 997af1195..5a061b2c9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -320,8 +320,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); - ValidPathInfo info(makeFixedOutputPath(method, hash, name)); - info.narHash = narHash; + ValidPathInfo info { + makeFixedOutputPath(method, hash, name), + narHash, + }; info.narSize = narSize; info.ca = FixedOutputHash { .method = method, .hash = hash }; @@ -849,19 +851,22 @@ void copyClosure(ref srcStore, ref dstStore, } -std::optional decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) { std::string path; getline(str, path); if (str.eof()) { return {}; } - ValidPathInfo info(store.parseStorePath(path)); - if (hashGiven) { + if (!hashGiven) { string s; getline(str, s); - info.narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); + uint64_t narSize; + if (!string2Int(s, narSize)) throw Error("number expected"); + hashGiven = { narHash, narSize }; } + ValidPathInfo info(store.parseStorePath(path), hashGiven->first); + info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e94d975c5..f6f6acaf5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -4,7 +4,6 @@ #include "hash.hh" #include "content-address.hh" #include "serialise.hh" -#include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" @@ -761,7 +760,7 @@ string showPaths(const PathSet & paths); std::optional decodeValidPathInfo( const Store & store, std::istream & str, - bool hashGiven = false); + std::optional hashGiven = std::nullopt); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index cc104bf35..e08d908e6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -495,7 +495,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) ValidPathInfos infos; while (1) { - auto info = decodeValidPathInfo(*store, cin, hashGiven); + // We use a dummy value because we'll set it below. FIXME be correct by + // construction and avoid dummy value. + auto hashResultOpt = !hashGiven ? std::optional { {Hash::dummy, -1} } : std::nullopt; + auto info = decodeValidPathInfo(*store, cin, hashResultOpt); if (!info) break; if (!store->isValidPath(info->path) || reregister) { /* !!! races */ @@ -944,11 +947,13 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("importing paths is not allowed"); auto path = readString(in); - ValidPathInfo info(store->parseStorePath(path)); auto deriver = readString(in); + ValidPathInfo info { + store->parseStorePath(path), + Hash::parseAny(readString(in), htSHA256), + }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash::parseAny(readString(in), htSHA256); info.references = readStorePaths(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index e183cb8b5..713155840 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -60,8 +60,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand hash = hsink.finish().first; } - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); - info.narHash = narHash; + ValidPathInfo info { + store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + narHash, + }; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { .method = ingestionMethod, diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 712043978..38b60fc38 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -77,10 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path); - info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c6cd88c49..cffc9ee44 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -129,9 +129,11 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references)); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + narHash, + }; info.references = std::move(references); - info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; From 641c95070162595c71a1f775a2364cd8533e1c4b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 6 Aug 2020 18:13:14 -0500 Subject: [PATCH 187/207] Add hashed-mirrors back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some users have their own hashed-mirrors setup, that is used to mirror things in addition to what’s available on tarballs.nixos.org. Although this should be feasable to do with a Binary Cache, it’s not always easy, since you have to remember what "name" each of the tarballs has. Continuing to support hashed-mirrors is cheap, so it’s best to leave support in Nix. Note that NIX_HASHED_MIRRORS is also supported in Nixpkgs through fetchurl.nix. Note that this excludes tarballs.nixos.org from the default, as in \#3689. All of these are available on cache.nixos.org. --- doc/manual/command-ref/conf-file.xml | 27 +++++++++++++++++++++++++++ src/libstore/builtins/fetchurl.cc | 14 ++++++++++++++ src/libstore/globals.hh | 3 +++ 3 files changed, 44 insertions(+) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 9c55526a3..d0f1b09ca 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -370,6 +370,33 @@ false. + hashed-mirrors + + A list of web servers used by + builtins.fetchurl to obtain files by hash. + Given a hash type ht and a base-16 hash + h, Nix will try to download the file + from + hashed-mirror/ht/h. + This allows files to be downloaded even if they have disappeared + from their original URI. For example, given the hashed mirror + http://tarballs.example.com/, when building the + derivation + + +builtins.fetchurl { + url = "https://example.org/foo-1.2.3.tar.xz"; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; +} + + + Nix will attempt to download this file from + http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae + first. If it is not available there, if will try the original URI. + + + + http-connections The maximum number of parallel TCP connections diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 6585a480d..2048f8f87 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,6 +58,20 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; + /* Try the hashed mirrors first. */ + if (getAttr("outputHashMode") == "flat") + for (auto hashedMirror : settings.hashedMirrors.get()) + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + auto ht = parseHashType(getAttr("outputHashAlgo")); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } + + /* Otherwise try the specified URL. */ fetch(mainUrl); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3406a9331..e3bb4cf84 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -335,6 +335,9 @@ public: "setuid/setgid bits or with file capabilities."}; #endif + Setting hashedMirrors{this, {}, "hashed-mirrors", + "A list of servers used by builtins.fetchurl to fetch files by hash."}; + Setting minFree{this, 0, "min-free", "Automatically run the garbage collector when free disk space drops below the specified amount."}; From 96c158d6e1c6c3bed6451a3c6447d2b0bd0337ad Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 6 Aug 2020 19:00:33 -0500 Subject: [PATCH 188/207] Fix build --- src/libstore/builtins/fetchurl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 2048f8f87..4fb5d8a06 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,8 +63,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); + std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { From 2ffc0589507de63bea5555ba993ac195154b4517 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Aug 2020 14:13:24 +0200 Subject: [PATCH 189/207] Make --no-eval-cache a global setting --- src/libexpr/eval.hh | 3 +++ src/nix/app.cc | 2 +- src/nix/flake.cc | 11 ++--------- src/nix/installables.cc | 22 +++++++++------------- src/nix/installables.hh | 9 ++++----- src/nix/search.cc | 2 +- 6 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0382298b3..5855b4ef2 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -375,6 +375,9 @@ struct EvalSettings : Config Setting traceFunctionCalls{this, false, "trace-function-calls", "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; }; extern EvalSettings evalSettings; diff --git a/src/nix/app.cc b/src/nix/app.cc index 3935297cf..80acbf658 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -8,7 +8,7 @@ namespace nix { App Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state, true); + auto [cursor, attrPath] = getCursor(state); auto type = cursor->getAttr("type")->getString(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 80d8654bc..653f8db1b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -572,7 +572,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand Strings{templateName == "" ? "defaultTemplate" : templateName}, Strings(attrsPathPrefixes), lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState, true); + auto [cursor, attrPath] = installable.getCursor(*evalState); auto templateDir = cursor->getAttr("path")->getString(); @@ -782,7 +782,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand { bool showLegacy = false; - bool useEvalCache = true; CmdFlakeShow() { @@ -791,12 +790,6 @@ struct CmdFlakeShow : FlakeCommand .description = "show the contents of the 'legacyPackages' output", .handler = {&showLegacy, true} }); - - addFlag({ - .longName = "no-eval-cache", - .description = "do not use the flake evaluation cache", - .handler = {[&]() { useEvalCache = false; }} - }); } std::string description() override @@ -934,7 +927,7 @@ struct CmdFlakeShow : FlakeCommand } }; - auto cache = openEvalCache(*state, flake, useEvalCache); + auto cache = openEvalCache(*state, flake); visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 06e61380b..ea152709f 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -183,8 +183,7 @@ void completeFlakeRefWithFragment( auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags)), - true); + std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -273,7 +272,7 @@ Buildable Installable::toBuildable() } std::vector, std::string>> -Installable::getCursors(EvalState & state, bool useEvalCache) +Installable::getCursors(EvalState & state) { auto evalCache = std::make_shared(std::nullopt, state, @@ -282,9 +281,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache) } std::pair, std::string> -Installable::getCursor(EvalState & state, bool useEvalCache) +Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state, useEvalCache); + auto cursors = getCursors(state); if (cursors.empty()) throw Error("cannot find flake attribute '%s'", what()); return cursors[0]; @@ -420,12 +419,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked ref openEvalCache( EvalState & state, - std::shared_ptr lockedFlake, - bool useEvalCache) + std::shared_ptr lockedFlake) { auto fingerprint = lockedFlake->getFingerprint(); return make_ref( - useEvalCache && evalSettings.pureEval + evalSettings.useEvalCache && evalSettings.pureEval ? std::optional { std::cref(fingerprint) } : std::nullopt, state, @@ -460,10 +458,9 @@ static std::string showAttrPaths(const std::vector & paths) std::tuple InstallableFlake::toDerivation() { - auto lockedFlake = getLockedFlake(); - auto cache = openEvalCache(*state, lockedFlake, true); + auto cache = openEvalCache(*state, lockedFlake); auto root = cache->getRoot(); for (auto & attrPath : getActualAttrPaths()) { @@ -517,11 +514,10 @@ std::pair InstallableFlake::toValue(EvalState & state) } std::vector, std::string>> -InstallableFlake::getCursors(EvalState & state, bool useEvalCache) +InstallableFlake::getCursors(EvalState & state) { auto evalCache = openEvalCache(state, - std::make_shared(lockFlake(state, flakeRef, lockFlags)), - useEvalCache); + std::make_shared(lockFlake(state, flakeRef, lockFlags))); auto root = evalCache->getRoot(); diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 9edff3331..26e87ee3a 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -62,10 +62,10 @@ struct Installable } virtual std::vector, std::string>> - getCursors(EvalState & state, bool useEvalCache); + getCursors(EvalState & state); std::pair, std::string> - getCursor(EvalState & state, bool useEvalCache); + getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const { @@ -118,7 +118,7 @@ struct InstallableFlake : InstallableValue std::pair toValue(EvalState & state) override; std::vector, std::string>> - getCursors(EvalState & state, bool useEvalCache) override; + getCursors(EvalState & state) override; std::shared_ptr getLockedFlake() const; @@ -127,7 +127,6 @@ struct InstallableFlake : InstallableValue ref openEvalCache( EvalState & state, - std::shared_ptr lockedFlake, - bool useEvalCache); + std::shared_ptr lockedFlake); } diff --git a/src/nix/search.cc b/src/nix/search.cc index 65a1e1818..430979274 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -177,7 +177,7 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state, true)) + for (auto & [cursor, prefix] : installable->getCursors(*state)) visit(*cursor, parseAttrPath(*state, prefix)); if (!json && !results) From 3c75ddc16b054a7dabf0710ee0a4323b4371effd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Aug 2020 14:46:00 +0200 Subject: [PATCH 190/207] nix build (and others): Force re-evaluation of cached errors Fixes #3872. This is a bit hacky. Ideally we would automatically re-evaluate the failed attribute iff we need to print the error message (so in commands like 'nix search' we wouldn't re-evaluate because we're suppressing errors). --- src/libexpr/eval-cache.cc | 17 ++++++++++------- src/libexpr/eval-cache.hh | 6 ++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index deb32484f..46177a0a4 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -405,7 +405,7 @@ Value & AttrCursor::forceValue() return v; } -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { if (!cachedValue) @@ -422,9 +422,12 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) if (attr) { if (std::get_if(&attr->second)) return nullptr; - else if (std::get_if(&attr->second)) - throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - else + else if (std::get_if(&attr->second)) { + if (forceErrors) + debug("reevaluating failed cached attribute '%s'"); + else + throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + } else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } @@ -469,9 +472,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -std::shared_ptr AttrCursor::getAttr(Symbol name) +std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors) { - auto p = maybeGetAttr(name); + auto p = maybeGetAttr(name, forceErrors); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return p; @@ -600,7 +603,7 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath); + auto aDrvPath = getAttr(root->state.sDrvPath, true); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index afee85fa9..8ffffc0ed 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -9,6 +9,8 @@ namespace nix::eval_cache { +MakeError(CachedEvalError, EvalError); + class AttrDb; class AttrCursor; @@ -92,11 +94,11 @@ public: std::string getAttrPathStr(Symbol name) const; - std::shared_ptr maybeGetAttr(Symbol name); + std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); std::shared_ptr maybeGetAttr(std::string_view name); - std::shared_ptr getAttr(Symbol name); + std::shared_ptr getAttr(Symbol name, bool forceErrors = false); std::shared_ptr getAttr(std::string_view name); From edfd676e059578fb574ce78d1a2cc66d018d3b16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Aug 2020 21:18:29 +0200 Subject: [PATCH 191/207] Fix .ls file names in binary caches These are not supposed to include the 'name' part of the store path. This was broken by 759947bf72. --- src/libstore/binary-cache-store.cc | 2 +- tests/binary-cache.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9682db730..86877dd1a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -219,7 +219,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } } - upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); } /* Optionally maintain an index of DWARF debug info files diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index b05a7d167..fe4ddec8d 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -219,7 +219,7 @@ outPath=$(nix-build --no-out-link -E ' nix copy --to file://$cacheDir?write-nar-listing=1 $outPath diff -u \ - <(jq -S < $cacheDir/$(basename $outPath).ls) \ + <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) From bcd0629c2e499788e4b3a9d6e380143a8f807419 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Aug 2020 20:32:35 +0000 Subject: [PATCH 192/207] Remove name parameter from `writeDerivation` The name is now stored with the derivation itself. --- src/libexpr/primops.cc | 2 +- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 2 +- src/nix/develop.cc | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 65d36ca0e..3955287b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -839,7 +839,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPath = writeDerivation(state.store, drv, state.repair); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 68b081058..f7e764a80 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -62,7 +62,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, - const Derivation & drv, std::string_view name, RepairFlag repair) + const Derivation & drv, RepairFlag repair) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) @@ -70,7 +70,7 @@ StorePath writeDerivation(ref store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - auto suffix = std::string(name) + drvExtension; + auto suffix = std::string(drv.name) + drvExtension; auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 14e0e947a..53dfc7f13 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -128,7 +128,7 @@ enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref store, - const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); + const Derivation & drv, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 12658078a..434088da7 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -124,10 +124,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = std::string(drvPath.name()); - assert(hasSuffix(drvName, ".drv")); - drvName.resize(drvName.size() - 4); - drvName += "-env"; + drv.name += "-env"; for (auto & output : drv.outputs) drv.env.erase(output.first); drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}}; @@ -136,12 +133,12 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.env["outputs"] = "out"; drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - auto shellOutPath = store->makeOutputPath("out", h, drvName); + auto shellOutPath = store->makeOutputPath("out", h, drv.name); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = shellOutPath } }); drv.env["out"] = store->printStorePath(shellOutPath); - auto shellDrvPath2 = writeDerivation(store, drv, drvName); + auto shellDrvPath2 = writeDerivation(store, drv); /* Build the derivation. */ store->buildPaths({{shellDrvPath2}}); From 581183d4d57d5382e3a375f760f3323ef451f555 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Aug 2020 20:51:34 +0000 Subject: [PATCH 193/207] Deduplicate parsing and reading derivations --- src/libstore/derivations.cc | 54 +++++++++++-------------------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f7e764a80..e9e4b5173 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -139,18 +139,14 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput(const Store & store, + StorePath path, std::string_view hashAlgo, std::string_view hash) { - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); const auto hash = parseString(str); - expect(str, ")"); - if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); + hashAlgo = hashAlgo.substr(2); } const HashType hashType = parseHashType(hashAlgo); @@ -178,6 +174,16 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings }; } +static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +{ + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); const auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + return parseDerivationOutput(store, std::move(path), hashAlgo, hash); +} + static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { @@ -541,38 +547,10 @@ StorePathSet BasicDerivation::outputPaths(const Store & store) const static DerivationOutput readDerivationOutput(Source & in, const Store & store) { auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); + const auto hashAlgo = readString(in); + const auto hash = readString(in); - if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (string(hashAlgo, 0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); - } - auto hashType = parseHashType(hashAlgo); - return hash != "" - ? DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, - } - } - : (settings.requireExperimentalFeature("ca-derivations"), - DerivationOutput { - .output = DerivationOutputCAFloating { - .method = std::move(method), - .hashType = std::move(hashType), - }, - }); - } else - return DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(path), - } - }; + return parseDerivationOutput(store, std::move(path), hashAlgo, hash); } StringSet BasicDerivation::outputNames() const From 5f8ae16c8b08485e7625283ccafb99727cd47693 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 10 Aug 2020 17:44:17 +0200 Subject: [PATCH 194/207] Always reset ANSI colors in progress-bar line When having a message like `waiting for a machine to build X` and building with `nix build -L`, the log-prefix is always colored yellow[1] on a small terminal-width as everything (including the ANSI color-reset) is stripped away. To work around that problem, this patch explicitly adds an `ANSI_NORMAL` to the end of the line. [1] https://imgur.com/a/FjtJOk3 --- src/libmain/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 3f7d99a1d..be3c06a38 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -362,7 +362,7 @@ public: auto width = getWindowSize().second; if (width <= 0) width = std::numeric_limits::max(); - writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); } std::string getStatus(State & state) From d3fa8c04c68a9532f85a18271c35457981a840ec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 11 Aug 2020 01:13:26 +0000 Subject: [PATCH 195/207] Simplify code as output env vars are unconditional Since the jsonObject unique ptr is reset to flush the string to make `__json`, all these `!jsonObject` conditions will always be true. --- src/libexpr/primops.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 65d36ca0e..af751a496 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -781,7 +781,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); - if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { .hash = FixedOutputHash { @@ -795,7 +795,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * else if (contentAddressed) { HashType ht = parseHashType(outputHashAlgo); for (auto & i : outputs) { - if (!jsonObject) drv.env[i] = hashPlaceholder(i); + drv.env[i] = hashPlaceholder(i); drv.outputs.insert_or_assign(i, DerivationOutput { .output = DerivationOutputCAFloating { .method = ingestionMethod, @@ -813,7 +813,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * that changes in the set of output names do get reflected in the hash. */ for (auto & i : outputs) { - if (!jsonObject) drv.env[i] = ""; + drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { .output = DerivationOutputInputAddressed { @@ -828,7 +828,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { auto outPath = state.store->makeOutputPath(i, h, drvName); - if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); + drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { .output = DerivationOutputInputAddressed { From 1a281ec07f33baa0d1b10d2760edcc7019caec83 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 11 Aug 2020 10:29:43 -0600 Subject: [PATCH 196/207] demote remote build message to Info --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5247cefa6..cec7b369b 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -201,7 +201,7 @@ static int _main(int argc, char * * argv) % concatStringsSep(", ", m.mandatoryFeatures); } - logError({ + logErrorInfo(lvlInfo, { .name = "Remote build", .description = "Failed to find a machine for remote build!", .hint = hint From 8d4162ff9e940ea9e2f97b07f3030a722695901a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Aug 2020 15:14:56 +0000 Subject: [PATCH 197/207] Separate auth and logic for the daemon Before, processConnection wanted to know a user name and user id, and `nix-daemon --stdio`, when it isn't proxying to an underlying daemon, would just assume "root" and 0. But `nix-daemon --stdio` (no proxying) shouldn't make guesses about who holds the other end of its standard streams. Now processConnection takes an "auth hook", so `nix-daemon` can provide the appropriate policy and daemon.cc doesn't need to know or care what it is. --- src/libstore/build.cc | 3 ++- src/libstore/daemon.cc | 13 ++----------- src/libstore/daemon.hh | 7 +++++-- src/nix-daemon/nix-daemon.cc | 15 +++++++++++++-- tests/remote-store.sh | 3 +++ 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 76baa1a6e..3fb052f00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2920,7 +2920,8 @@ void DerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + daemon::NotTrusted, daemon::Recursive, + [&](Store & store) { store.createUser("nobody", 65535); }); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5e568fc94..7a6eb99be 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -817,8 +817,7 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId) + std::function authHook) { auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -859,15 +858,7 @@ void processConnection( /* If we can't accept clientVersion, then throw an error *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - - store->createUser(userName, userId); + authHook(*store); tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 266932013..841ace316 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -12,7 +12,10 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId); + /* Arbitrary hook to check authorization / initialize user data / whatever + after the protocol has been negotiated. The idea is that this function + and everything it calls doesn't know about this stuff, and the + `nix-daemon` handles that instead. */ + std::function authHook); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index bcb86cbce..cfa634a44 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -239,7 +239,15 @@ static void daemonLoop(char * * argv) // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + store.createUser(user, peer.uid); + }); exit(0); }, options); @@ -324,7 +332,10 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limitting access to thoses is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } } else { daemonLoop(argv); diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 4cc73465a..3a61946f9 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -2,6 +2,9 @@ source common.sh clearStore +# Ensure "fake ssh" remote store works just as legacy fake ssh would. +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor + startDaemon storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh From 4720853129b6866775edd9f90ad6f10701f98a3c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Aug 2020 16:32:36 +0000 Subject: [PATCH 198/207] Make `system-features` a store setting This seems more correct. It also means one can specify the features a store should support with --store and remote-store=..., which is useful. I use this to clean up the build remotes test. --- src/build-remote/build-remote.cc | 16 ++++----------- src/libstore/build.cc | 8 ++++---- src/libstore/machines.cc | 24 ++++++++++++++++++++++ src/libstore/machines.hh | 4 ++++ src/libstore/parsed-derivations.cc | 8 ++++---- src/libstore/parsed-derivations.hh | 4 ++-- src/libstore/store-api.hh | 4 ++++ tests/build-hook.nix | 3 ++- tests/build-remote.sh | 33 +++++++++++++++++------------- tests/local.mk | 2 +- 10 files changed, 68 insertions(+), 38 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index cec7b369b..ce5127113 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -38,9 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static bool allSupportedLocally(const std::set& requiredFeatures) { +static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!settings.systemFeatures.get().count(feature)) return false; + if (!store.systemFeatures.get().count(feature)) return false; return true; } @@ -106,7 +106,7 @@ static int _main(int argc, char * * argv) auto canBuildLocally = amWilling && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) - && allSupportedLocally(requiredFeatures); + && allSupportedLocally(*store, requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -224,15 +224,7 @@ static int _main(int argc, char * * argv) Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); - Store::Params storeParams; - if (hasPrefix(bestMachine->storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; - if (bestMachine->sshKey != "") - storeParams["ssh-key"] = bestMachine->sshKey; - } - - sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore = bestMachine->openStore(); sshStore->connect(); storeUri = bestMachine->storeUri; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 76baa1a6e..d6e6ad6a9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1475,7 +1475,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1964,13 +1964,13 @@ void linkOrCopy(const Path & from, const Path & to) void DerivationGoal::startBuilder() { /* Right platform? */ - if (!parsedDrv->canBuildLocally()) + if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", worker.store.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); @@ -3179,7 +3179,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index f848582da..7db2556f4 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "util.hh" #include "globals.hh" +#include "store-api.hh" #include @@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set & features) const { }); } +ref Machine::openStore() const { + Store::Params storeParams; + if (hasPrefix(storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (sshKey != "") + storeParams["ssh-key"] = sshKey; + } + { + auto & fs = storeParams["system-features"]; + auto append = [&](auto feats) { + for (auto & f : feats) { + if (fs.size() > 0) fs += ' '; + fs += f; + } + }; + append(supportedFeatures); + append(mandatoryFeatures); + } + + return nix::openStore(storeUri, storeParams); +} + void parseMachines(const std::string & s, Machines & machines) { for (auto line : tokenizeString>(s, "\n;")) { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index de92eb924..341d9bd97 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -4,6 +4,8 @@ namespace nix { +class Store; + struct Machine { const string storeUri; @@ -28,6 +30,8 @@ struct Machine { decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + + ref openStore() const; }; typedef std::vector Machines; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 24f848e46..e7b7202d4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const return res; } -bool ParsedDerivation::canBuildLocally() const +bool ParsedDerivation::canBuildLocally(Store & localStore) const { if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform) @@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!settings.systemFeatures.get().count(feature)) return false; + if (!localStore.systemFeatures.get().count(feature)) return false; return true; } -bool ParsedDerivation::willBuildLocally() const +bool ParsedDerivation::willBuildLocally(Store & localStore) const { - return getBoolAttr("preferLocalBuild") && canBuildLocally(); + return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore); } bool ParsedDerivation::substitutesAllowed() const diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 6ee172d81..3fa09f34f 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -29,9 +29,9 @@ public: StringSet getRequiredSystemFeatures() const; - bool canBuildLocally() const; + bool canBuildLocally(Store & localStore) const; - bool willBuildLocally() const; + bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e94d975c5..1e940e6a8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -164,6 +164,10 @@ public: Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting systemFeatures{this, settings.systemFeatures, + "system-features", + "Optional features that the system this store builds on implements (like \"kvm\")."}; + protected: struct PathInfoCacheValue { diff --git a/tests/build-hook.nix b/tests/build-hook.nix index a19c10dde..1bd0b759f 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -23,6 +23,7 @@ let shell = busybox; name = "build-remote-input-2"; buildCommand = "echo BAR > $out"; + requiredSystemFeatures = ["bar"]; }; in @@ -34,6 +35,6 @@ in '' read x < ${input1} read y < ${input2} - echo $x$y > $out + echo "$x $y" > $out ''; } diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 4dfb753e1..7638f536f 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,31 +1,36 @@ source common.sh -clearStore - if ! canUseSandbox; then exit; fi if ! [[ $busybox =~ busybox ]]; then exit; fi -chmod -R u+w $TEST_ROOT/machine0 || true -chmod -R u+w $TEST_ROOT/machine1 || true -chmod -R u+w $TEST_ROOT/machine2 || true -rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2 -rm -f $TEST_ROOT/result - unset NIX_STORE_DIR unset NIX_STATE_DIR +function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } + +builders=( + # system-features will automatically be added to the outer URL, but not inner + # remote-store URL. + "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo" + "$TEST_ROOT/machine2 - - 1 1 bar" +) + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). +# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ --store $TEST_ROOT/machine0 \ - --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \ - --system-features foo + --builders "$(join_by '; ' "${builders[@]}")" outPath=$(readlink -f $TEST_ROOT/result) -cat $TEST_ROOT/machine0/$outPath | grep FOOBAR +grep 'FOO BAR' $TEST_ROOT/machine0/$outPath -# Ensure that input1 was built on store2 due to the required feature. -(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh) -nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh +# Ensure that input1 was built on store1 due to the required feature. +(! nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh) +nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh + +# Ensure that input2 was built on store2 due to the required feature. +(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-2.sh) +nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-2.sh diff --git a/tests/local.mk b/tests/local.mk index 0f3bfe606..5c77b9bb7 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,5 @@ nix_tests = \ - init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ + hash.sh lang.sh add.sh simple.sh dependencies.sh \ config.sh \ gc.sh \ gc-concurrent.sh \ From d2f2be0f701f8b091a00b8898dc7fb922096cfaf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Aug 2020 13:24:39 +0000 Subject: [PATCH 199/207] Test `RemoteStore::buildDerivation` Fix `wopNarFromPath` which needed a `toRealPath`. --- src/libstore/daemon.cc | 2 +- tests/build-hook.nix | 12 +++++++++++- tests/build-remote.sh | 23 ++++++++++++++++++----- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7a6eb99be..956a8dc8d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->printStorePath(path), to); + dumpPath(store->toRealPath(store->printStorePath(path)), to); break; } diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 1bd0b759f..eb16676f0 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -26,6 +26,16 @@ let requiredSystemFeatures = ["bar"]; }; + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + read x < ${input2} + echo $x BAZ > $out + ''; + requiredSystemFeatures = ["baz"]; + }; + in mkDerivation { @@ -34,7 +44,7 @@ in buildCommand = '' read x < ${input1} - read y < ${input2} + read y < ${input3} echo "$x $y" > $out ''; } diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 7638f536f..8833f4698 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -13,6 +13,7 @@ builders=( # remote-store URL. "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo" "$TEST_ROOT/machine2 - - 1 1 bar" + "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz" ) # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a @@ -25,12 +26,24 @@ nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ outPath=$(readlink -f $TEST_ROOT/result) -grep 'FOO BAR' $TEST_ROOT/machine0/$outPath +grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath + +set -o pipefail # Ensure that input1 was built on store1 due to the required feature. -(! nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh) -nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh +nix path-info --store $TEST_ROOT/machine1 --all \ + | grep builder-build-remote-input-1.sh \ + | grep -v builder-build-remote-input-2.sh \ + | grep -v builder-build-remote-input-3.sh # Ensure that input2 was built on store2 due to the required feature. -(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-2.sh) -nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-2.sh +nix path-info --store $TEST_ROOT/machine2 --all \ + | grep -v builder-build-remote-input-1.sh \ + | grep builder-build-remote-input-2.sh \ + | grep -v builder-build-remote-input-3.sh + +# Ensure that input3 was built on store3 due to the required feature. +nix path-info --store $TEST_ROOT/machine3 --all \ + | grep -v builder-build-remote-input-1.sh \ + | grep -v builder-build-remote-input-2.sh \ + | grep builder-build-remote-input-3.sh From 85aacbee64680e60b22e8a729756b0a3882f8b8f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Aug 2020 14:47:53 +0000 Subject: [PATCH 200/207] Use `TeeSink` and `TeeSouce` in a few more places --- src/libstore/binary-cache-store.cc | 12 ++++-------- src/libstore/local-store.cc | 6 +----- src/libstore/store-api.cc | 6 +++--- src/libutil/archive.cc | 6 +----- src/libutil/serialise.hh | 11 +++++++++++ 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 86877dd1a..c360b9dda 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -312,14 +312,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); - uint64_t narSize = 0; + LengthSink narSize; + TeeSink tee { sink, narSize }; - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - narSize += len; - }); - - auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + auto decompressor = makeDecompressionSink(info->compression, tee); try { getFile(info->url, *decompressor); @@ -331,7 +327,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME - stats.narReadBytes += narSize; + stats.narReadBytes += narSize.length; } void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3c66a4dfd..e552bdc59 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1021,11 +1021,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, else hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); - LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { - size_t n = source.read(data, len); - (*hashSink)(data, n); - return n; - }); + TeeSource wrapperSource { source, *hashSink }; restorePath(realPath, wrapperSource); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2837910b9..3d07e2d38 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -746,12 +746,12 @@ void copyStorePath(ref srcStore, ref dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); + LambdaSink progressSink([&](const unsigned char * data, size_t len) { total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath(storePath, wrapperSink); + TeeSink tee { sink, progressSink }; + srcStore->narFromPath(storePath, tee); }, [&]() { throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index ce7cf9754..14399dea3 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -366,11 +366,7 @@ void copyNAR(Source & source, Sink & sink) ParseSink parseSink; /* null sink; just parse the NAR */ - LambdaSource wrapper([&](unsigned char * data, size_t len) { - auto n = source.read(data, len); - sink(data, n); - return n; - }); + TeeSource wrapper { source, sink }; parseDump(parseSink, wrapper); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index c29c6b29b..69ae0874a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -225,6 +225,17 @@ struct SizedSource : Source } }; +/* A sink that that just counts the number of bytes given to it */ +struct LengthSink : Sink +{ + uint64_t length = 0; + + virtual void operator () (const unsigned char * _, size_t len) + { + length += len; + } +}; + /* Convert a function into a sink. */ struct LambdaSink : Sink { From 5ccd94501dac232cc09fb5301c4406cef72c0a27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 11 Aug 2020 13:43:29 +0000 Subject: [PATCH 201/207] Allow trustless building of CA derivations Include a long comment explaining the policy. Perhaps this can be moved to the manual at some point in the future. Also bump the daemon protocol minor version, so clients can tell whether `wopBuildDerivation` supports trustless CA derivation building. I hope to take advantage of this in a follow-up PR to support trustless remote building with the minimal sending of derivation closures. --- src/libstore/daemon.cc | 42 +++++++++++++++++++++++++++++++-- src/libstore/worker-protocol.hh | 2 +- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 956a8dc8d..45e81c8da 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -454,8 +454,46 @@ static void performOp(TunnelLogger * logger, ref store, readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); + + /* Content-addressed derivations are trustless because their output paths + are verified by their content alone, so any derivation is free to + try to produce such a path. + + Input-addressed derivation output paths, however, are calculated + from the derivation closure that produced them---even knowing the + root derivation is not enough. That the output data actually came + from those derivations is fundamentally unverifiable, but the daemon + trusts itself on that matter. The question instead is whether the + submitted plan has rights to the output paths it wants to fill, and + at least the derivation closure proves that. + + It would have been nice if input-address algorithm merely depended + on the build time closure, rather than depending on the derivation + closure. That would mean input-addressed paths used at build time + would just be trusted and not need their own evidence. This is in + fact fine as the same guarantees would hold *inductively*: either + the remote builder has those paths and already trusts them, or it + needs to build them too and thus their evidence must be provided in + turn. The advantage of this variant algorithm is that the evidence + for input-addressed paths which the remote builder already has + doesn't need to be sent again. + + That said, now that we have floating CA derivations, it is better + that people just migrate to those which also solve this problem, and + others. It's the same migration difficulty with strictly more + benefit. + + Lastly, do note that when we parse fixed-output content-addressed + derivations, we throw out the precomputed output paths and just + store the hashes, so there aren't two competing sources of truth an + attacker could exploit. */ + if (drv.type() == DerivationType::InputAddressed && !trusted) + throw Error("you are not privileged to build input-addressed derivations"); + + /* Make sure that the non-input-addressed derivations that got this far + are in fact content-addressed if we don't trust them. */ + assert(derivationIsCA(drv.type()) || trusted); + auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f76b13fb4..5eddaff56 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x117 +#define PROTOCOL_VERSION 0x118 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From e1308b121169ea8327c95556668ad4f7f4815402 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Aug 2020 03:13:17 +0000 Subject: [PATCH 202/207] Define `LegacySSHStore::buildPaths` using `cmdBuildPaths` Evidentally this was never implemented because Nix switched to using `buildDerivation` exclusively before `build-remote.pl` was rewritten. The `nix-copy-ssh` test (already) tests this. --- src/libstore/legacy-ssh-store.cc | 53 ++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c6eeab548..b5ece22f4 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -202,6 +202,24 @@ struct LegacySSHStore : public Store const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } +private: + + void putBuildSettings(Connection & conn) + { + conn.to + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) + conn.to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) + conn.to + << settings.buildRepeat + << settings.enforceDeterminism; + } + +public: + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { @@ -211,16 +229,8 @@ struct LegacySSHStore : public Store << cmdBuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); - conn->to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) - conn->to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->to - << settings.buildRepeat - << settings.enforceDeterminism; + + putBuildSettings(*conn); conn->to.flush(); @@ -234,6 +244,29 @@ struct LegacySSHStore : public Store return status; } + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + { + auto conn(connections->get()); + + conn->to << cmdBuildPaths; + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } + } + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } From ed026f7206a3154ce11bddac2e58541327313f6f Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 13 Aug 2020 17:44:42 -0700 Subject: [PATCH 203/207] Don't try to parse signature check as commit timestamp When the log.showSignature git setting is enabled, the output of "git log" contains signature verification information in addition to the timestamp GitInputScheme::fetch wants: $ git log -1 --format=%ct gpg: Signature made Sat 07 Sep 2019 02:02:03 PM PDT gpg: using RSA key 0123456789ABCDEF0123456789ABCDEF01234567 gpg: issuer "user@example.com" gpg: Good signature from "User " [ultimate] 1567890123 1567890123 For folks that had log.showSignature set, this caused all nix operations on flakes to fail: $ nix build error: stoull --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8b6e047f1..5ca0f8521 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -269,7 +269,7 @@ struct GitInputScheme : InputScheme // modified dirty file? input.attrs.insert_or_assign( "lastModified", - haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return { Tree(store->printStorePath(storePath), std::move(storePath)), @@ -421,7 +421,7 @@ struct GitInputScheme : InputScheme auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, From 4b571ea3216715ac1f2c06d1b0d68f27c6070d28 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Aug 2020 11:52:37 -0400 Subject: [PATCH 204/207] Update src/libstore/daemon.cc Co-authored-by: Eelco Dolstra --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 956a8dc8d..80ed64f02 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(store->printStorePath(path)), to); + dumpPath(store->toRealPath(path)), to); break; } From 6f7ac5e8658aad4c0a44c220d9a2b06ea4564980 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Aug 2020 21:59:31 +0000 Subject: [PATCH 205/207] Remove extra closing paren --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 80ed64f02..dde4122d1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path)), to); + dumpPath(store->toRealPath(path), to); break; } From dbf96e10ecc75410c9db798f208f8a8310842a4f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Aug 2020 17:38:12 +0000 Subject: [PATCH 206/207] Test remote building with fixed output derivations --- tests/build-hook-ca.nix | 45 +++++++++++++++++++ tests/build-remote-content-addressed-fixed.sh | 5 +++ tests/build-remote-input-addressed.sh | 5 +++ tests/build-remote.sh | 4 +- tests/local.mk | 3 +- 5 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 tests/build-hook-ca.nix create mode 100644 tests/build-remote-content-addressed-fixed.sh create mode 100644 tests/build-remote-input-addressed.sh diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca.nix new file mode 100644 index 000000000..98db473fc --- /dev/null +++ b/tests/build-hook-ca.nix @@ -0,0 +1,45 @@ +{ busybox }: + +with import ./config.nix; + +let + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo FOO > $out"; + requiredSystemFeatures = ["foo"]; + outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; + }; + + input2 = mkDerivation { + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo BAR > $out"; + requiredSystemFeatures = ["bar"]; + outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; + }; + +in + + mkDerivation { + shell = busybox; + name = "build-remote"; + buildCommand = + '' + read x < ${input1} + read y < ${input2} + echo "$x $y" > $out + ''; + outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0="; + } diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh new file mode 100644 index 000000000..1408a19d5 --- /dev/null +++ b/tests/build-remote-content-addressed-fixed.sh @@ -0,0 +1,5 @@ +source common.sh + +file=build-hook-ca.nix + +source build-remote.sh diff --git a/tests/build-remote-input-addressed.sh b/tests/build-remote-input-addressed.sh new file mode 100644 index 000000000..b34caa061 --- /dev/null +++ b/tests/build-remote-input-addressed.sh @@ -0,0 +1,5 @@ +source common.sh + +file=build-hook.nix + +source build-remote.sh diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 7638f536f..d9048583f 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,5 +1,3 @@ -source common.sh - if ! canUseSandbox; then exit; fi if ! [[ $busybox =~ busybox ]]; then exit; fi @@ -18,7 +16,7 @@ builders=( # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). -nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ +nix build -L -v -f $file -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ --store $TEST_ROOT/machine0 \ --builders "$(join_by '; ' "${builders[@]}")" diff --git a/tests/local.mk b/tests/local.mk index 5c77b9bb7..492d6a0fd 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,8 @@ nix_tests = \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ build-dry.sh \ - build-remote.sh \ + build-remote-input-addressed.sh \ + build-remote-content-addressed-fixed.sh \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ From 07975979aae4e7729ae13ffeb7390d07d71ad4bd Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 17 Aug 2020 15:04:54 -0400 Subject: [PATCH 207/207] Comment out fixed content address test --- tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index 492d6a0fd..53035da41 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -15,7 +15,6 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ - build-remote-content-addressed-fixed.sh \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ @@ -35,6 +34,7 @@ nix_tests = \ recursive.sh \ flakes.sh # parallel.sh + # build-remote-content-addressed-fixed.sh \ install-tests += $(foreach x, $(nix_tests), tests/$(x))