From 79c169d1c615211af69c1cbc6218f7465a4f81ed Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:49:09 -0500 Subject: [PATCH 01/70] Allow substituting from different storeDir Substituters can substitute from one store dir to another with a little bit of help. The store api just needs to have a CA so it can recompute the store path based on the new store dir. We can only do this for fixed output derivations with no references, though. --- src/libstore/local-store.cc | 16 ++++++++++++---- src/libstore/local-store.hh | 3 ++- src/libstore/misc.cc | 27 ++++++++++++++++++++++++++- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 7 +++++++ src/libstore/store-api.hh | 2 +- 7 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..6385453cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; - for (auto & path : paths) { - if (infos.count(path)) continue; + for (auto & path_ : paths) { + auto path(path_.clone()); debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); + + auto ca = pathsCA.find(printStorePath(path)); + if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) { + if (!hasPrefix(ca->second, "fixed:")) + continue; + // recompute store path so that we can use a fixed output ca + path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); + } else continue; + auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..5a976e3e1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -142,7 +142,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, + std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..1b0a055d3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath, } +static std::optional getDerivationCA(ref drv) +{ + auto outputHashMode = drv->env.find("outputHashMode"); + auto outputHashAlgo = drv->env.find("outputHashAlgo"); + auto outputHash = drv->env.find("outputHash"); + if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) { + auto ht = parseHashType(outputHashAlgo->second); + auto h = Hash(outputHash->second, ht); + FileIngestionMethod ingestionMethod; + if (outputHashMode->second == "recursive") + ingestionMethod = FileIngestionMethod::Recursive; + else if (outputHashMode->second == "flat") + ingestionMethod = FileIngestionMethod::Flat; + else + throw Error("unknown ingestion method: '%s'", outputHashMode->second); + return makeFixedOutputCA(ingestionMethod, h); + } + + return std::nullopt; +} + void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) @@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector & targets, SubstitutablePathInfos infos; StorePathSet paths; // FIXME paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + + std::map pathsCA = {}; + if (auto ca = getDerivationCA(drv)) + pathsCA.insert({outPathS, *ca}); + querySubstitutablePathInfos(paths, infos, pathsCA); if (infos.empty()) { drvState_->lock()->done = true; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..956980f49 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (paths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..867d650bc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -58,7 +58,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..40000da01 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -580,6 +580,13 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; + // recompute store path on the chance dstStore does it differently + if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { + auto info2 = make_ref(*info); + info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name()); + info = info2; + } + if (!info->narHash) { StringSink sink; srcStore->narFromPath({storePath}, sink); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..6f961329c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -445,7 +445,7 @@ public: sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos, std::map pathsCA = {}) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, From 3e3eaa90dd1aaf4684697f27726ec72aec75206c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:51:44 -0500 Subject: [PATCH 02/70] Remove hashed-mirrors --- doc/manual/command-ref/conf-file.xml | 28 ---------------------------- src/libstore/builtins/fetchurl.cc | 14 -------------- src/libstore/globals.hh | 3 --- 3 files changed, 45 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 1fa74a143..9c55526a3 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -370,34 +370,6 @@ false. - hashed-mirrors - - A list of web servers used by - builtins.fetchurl to obtain files by - hash. The default is - http://tarballs.nixos.org/. Given a hash type - ht and a base-16 hash - h, Nix will try to download the file - from - hashed-mirror/ht/h. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the default mirror - http://tarballs.nixos.org/, when building the derivation - - -builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} - - - Nix will attempt to download this file from - http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae - first. If it is not available there, if will try the original URI. - - - - http-connections The maximum number of parallel TCP connections diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 486babf14..254cb4fce 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,20 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); - } - - /* Otherwise try the specified URL. */ fetch(mainUrl); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..2e1e405b3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -339,9 +339,6 @@ public: "setuid/setgid bits or with file capabilities."}; #endif - Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - Setting minFree{this, 0, "min-free", "Automatically run the garbage collector when free disk space drops below the specified amount."}; From 11c97070f3d845a9313f137c41974c021429e834 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 10:14:03 -0500 Subject: [PATCH 03/70] Fix storeDir != storeDir condition this needs to only continue if the path replacement fails. --- src/libstore/local-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6385453cc..4cecf0992 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -866,12 +866,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto ca = pathsCA.find(printStorePath(path)); - if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) { + if (info->references.empty() && ca != pathsCA.end()) { if (!hasPrefix(ca->second, "fixed:")) continue; - // recompute store path so that we can use a fixed output ca + // recompute store path so that we can use a different store path path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); - } else continue; + } else if (sub->storeDir != storeDir) continue; auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); From 1c55f16a1648431301f044710644a674c018321d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 12:26:08 -0500 Subject: [PATCH 04/70] Add --flat to nix add-to-store This can be used to add flat hashes to the nix store. --- src/nix/add-to-store.cc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..a5a77f17c 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "flat", + .shortName = 0, + .description = "use flat file ingestion", + .handler = {&ingestionMethod, FileIngestionMethod::Flat}, + }); } std::string description() override @@ -45,10 +53,24 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + + Hash hash; + switch (ingestionMethod) { + case FileIngestionMethod::Recursive: { + hash = info.narHash; + break; + } + case FileIngestionMethod::Flat: { + HashSink hsink(htSHA256); + readFile(path, hsink); + hash = hsink.finish().first; + break; + } + } + info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) { auto source = StringSource { *sink.s }; From 2f2ac850b54110a81e3338468161cf132618c156 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:03:17 -0500 Subject: [PATCH 05/70] Compute new store path correctly --- src/libstore/local-store.cc | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4cecf0992..bf005a3b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -859,19 +859,28 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - for (auto & path_ : paths) { - auto path(path_.clone()); - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); - try { - auto info = sub->queryPathInfo(path); + for (auto & path : paths) { + auto subPath(path.clone()); - auto ca = pathsCA.find(printStorePath(path)); - if (info->references.empty() && ca != pathsCA.end()) { - if (!hasPrefix(ca->second, "fixed:")) - continue; - // recompute store path so that we can use a different store path - path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); - } else if (sub->storeDir != storeDir) continue; + auto ca_ = pathsCA.find(printStorePath(path)); + // recompute store path so that we can use a different store root + if (ca_ != pathsCA.end()) { + auto ca(ca_->second); + if (!hasPrefix(ca, "fixed:")) + continue; + FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + subPath = makeFixedOutputPath(ingestionMethod, hash, path.name()); + if (subPath != path) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !info->isContentAddressed(*sub)) + continue; auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); From 9077b9e6b28b94cdfac80ea4afbe18d13a7b2de6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:27:28 -0500 Subject: [PATCH 06/70] Separate dstStore path from srcStore path --- src/libstore/store-api.cc | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 40000da01..1b340bf6c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -649,15 +649,24 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st processGraph(pool, PathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const Path & storePathS) { + auto storePath = srcStore->parseStorePath(storePathS); + + auto info = srcStore->queryPathInfo(storePath); + auto storePathForDst = storePath.clone(); + if (hasPrefix(info->ca, "fixed:")) + { + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + } + + if (dstStore->isValidPath(storePathForDst)) { nrDone++; showProgress(); return PathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); @@ -667,9 +676,18 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st [&](const Path & storePathS) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); + auto storePath = srcStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); - if (!dstStore->isValidPath(storePath)) { + auto storePathForDst = storePath.clone(); + if (hasPrefix(info->ca, "fixed:")) + { + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + } + + if (!dstStore->isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); showProgress(); try { From 1c148a80fe05580f94e624134b923799f6cbc7b3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:39:44 -0500 Subject: [PATCH 07/70] Replace --hashed-mirrors with substituters test --- tests/check.sh | 14 +++++++------- tests/fetchurl.sh | 21 +++++++++------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index 5f25d04cb..5f4997e28 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l [ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log -path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') +path=$(nix-build check.nix -A fetchurl --no-out-link) chmod +w $path echo foo > $path chmod -w $path -nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --check # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] -nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] echo -n > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +nix-build check.nix -A hashmismatch --no-out-link echo 'Hello World' > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] # Multiple failures with --keep-going nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$? [ "$status" = "110" ] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 2535651b0..13447d774 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,7 +5,7 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -14,7 +14,7 @@ clearStore hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -25,26 +25,23 @@ hash=$(nix hash-file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) cmp $outPath fetchurl.sh -# Test the hashed mirror feature. +# Test that we can substitute from a different store dir. clearStore -hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) +other_store=file://$TEST_ROOT/other_store -mirror=$TEST_ROOT/hashed-mirror -rm -rf $mirror -mkdir -p $mirror/sha512 -ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 +hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) +storePath=$(nix add-to-store --store $other_store ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +outPath=$(nix-build -vvvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ - --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + --argstr name bla --no-out-link --substituters $other_store # Test unpacking a NAR. rm -rf $TEST_ROOT/archive From 85d01e1f658d93a0ea8d0396b1e79d03c2ff1da2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:53:30 -0500 Subject: [PATCH 08/70] =?UTF-8?q?Don=E2=80=99t=20use=20makeStorePath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1b340bf6c..afa557e16 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -583,7 +583,9 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name()); + FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); info = info2; } From 5b386b05f58b1ed996afe47791d85e6d20da91a2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 14:57:45 -0500 Subject: [PATCH 09/70] Recompute storePath based on isContentAddressed --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index afa557e16..c8dc4744f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -581,7 +581,7 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { + if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); From e3cb536f19920018b522f76ac2661d68627b7c9e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:25:29 -0500 Subject: [PATCH 10/70] Fix add-to-store --flat to put in correct hash --- src/nix/add-to-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index a5a77f17c..26edcf95a 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -53,14 +53,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart)); - info.narHash = narHash; - info.narSize = sink.s->size(); - Hash hash; switch (ingestionMethod) { case FileIngestionMethod::Recursive: { - hash = info.narHash; + hash = narHash; break; } case FileIngestionMethod::Flat: { @@ -70,6 +66,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand break; } } + + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + info.narHash = narHash; + info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(ingestionMethod, hash); if (!dryRun) { From 88120442d23a7e00833e8b0d523a6aa8072287b3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:32:52 -0500 Subject: [PATCH 11/70] Debug when storePath changes these rewrites should be transparent, but they are important to know about when debugging --- src/libstore/store-api.cc | 4 ++++ tests/fetchurl.sh | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8dc4744f..7cb48b293 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -661,6 +661,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (storePathForDst != storePath) + debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (dstStore->isValidPath(storePathForDst)) { @@ -687,6 +689,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (storePathForDst != storePath) + debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (!dstStore->isValidPath(storePathForDst)) { diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 13447d774..510f98843 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -35,12 +35,13 @@ clearStore other_store=file://$TEST_ROOT/other_store hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix add-to-store --store $other_store ./fetchurl.sh) -outPath=$(nix-build -vvvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) + +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ --argstr name bla --no-out-link --substituters $other_store # Test unpacking a NAR. From 006f1252d2fdb36c7ee66bf79f0df2ab8cd16816 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 15:33:24 -0500 Subject: [PATCH 12/70] Fix SRI test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can’t use custom name here because different names will have different store paths. This is a limitation of the Store API’s reliance on store paths. We might be able to get around the above in the future by using a dummy name for certain fixed output paths. --- tests/fetchurl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 510f98843..f4b86d251 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,7 +42,7 @@ outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchu # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ - --argstr name bla --no-out-link --substituters $other_store + --no-out-link --substituters $other_store # Test unpacking a NAR. rm -rf $TEST_ROOT/archive From b2cb288cdd462cbca35ffd3afda9d7ec2d595209 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 16:36:35 -0500 Subject: [PATCH 13/70] Add makeFixedOutputPathFromCA function This puts what we are already doing into a shared method. It just needs a path name and a ca and produces a store path. --- src/libstore/local-store.cc | 11 +++-------- src/libstore/store-api.cc | 31 ++++++++++++++++++------------- src/libstore/store-api.hh | 6 +++++- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bf005a3b3..9158a52e0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -862,15 +862,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, for (auto & path : paths) { auto subPath(path.clone()); - auto ca_ = pathsCA.find(printStorePath(path)); + auto ca = pathsCA.find(printStorePath(path)); // recompute store path so that we can use a different store root - if (ca_ != pathsCA.end()) { - auto ca(ca_->second); - if (!hasPrefix(ca, "fixed:")) - continue; - FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - subPath = makeFixedOutputPath(ingestionMethod, hash, path.name()); + if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { + subPath = makeFixedOutputPathFromCA(path.name(), ca->second); if (subPath != path) debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7cb48b293..bc9a6c4f5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -191,6 +191,19 @@ StorePath Store::makeFixedOutputPath( } } +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, std::string ca, + const StorePathSet & references, bool hasSelfReference) const +{ + if (hasPrefix(ca, "fixed:")) { + FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); + return makeFixedOutputPath(ingestionMethod, hash, name, references, hasSelfReference); + } else if (hasPrefix(ca, "text:")) { + Hash hash(std::string(ca, 5)); + return makeTextPath(name, hash, references); + } else + throw Error("'%s' is not a valid ca", ca); +} StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const @@ -583,9 +596,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); info = info2; } @@ -656,11 +667,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath.clone(); - if (hasPrefix(info->ca, "fixed:")) - { - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (info->isContentAddressed(*srcStore)) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } @@ -684,11 +692,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath.clone(); - if (hasPrefix(info->ca, "fixed:")) - { - FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name()); + if (info->isContentAddressed(*srcStore)) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6f961329c..9412785b4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -352,7 +352,11 @@ public: bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, std::string ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store From c214cda9401cf50d0419038746428260b0dfdd63 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 13 Jun 2020 00:07:42 -0500 Subject: [PATCH 14/70] Correctly substitute from different storeDir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Originally, the test was only checking for different “real” storeDir. That’s an easy case to handle, but the much harder one is if different virtual store dirs are used. To do this, we need the SubstitutionGoal to know about the ca, so it can recalculate the path to copy it over. An important note here is that the store path passed to copyStorePath needs to be one for srcStore - so that queryPathInfo works properly. This also adds an error message when the store path from queryPathInfo is different from the one we requested. --- src/libstore/build.cc | 44 ++++++++++++++++++++++++++++----------- src/libstore/misc.cc | 12 +++++------ src/libstore/store-api.hh | 2 ++ tests/fetchurl.sh | 4 ++-- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..f3f6f01cc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -305,7 +305,7 @@ public: GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1195,7 +1195,7 @@ void DerivationGoal::haveDerivation() them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv))); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -4268,8 +4268,11 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* Content address for recomputing store path */ + std::optional ca; + public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut() override { abort(); }; @@ -4304,10 +4307,11 @@ public: }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(std::move(storePath)) , repair(repair) + , ca(ca) { state = &SubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); @@ -4382,14 +4386,13 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } + auto subPath = storePath.clone(); + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); try { // FIXME: make async - info = sub->queryPathInfo(storePath); + info = sub->queryPathInfo(subPath); } catch (InvalidPath &) { tryNext(); return; @@ -4408,6 +4411,19 @@ void SubstitutionGoal::tryNext() throw; } + if (info->path != storePath) { + if (info->isContentAddressed(*sub)) { + auto info2 = std::const_pointer_cast(std::shared_ptr(info)); + info2->path = storePath.clone(); + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + tryNext(); + return; + } + } + /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast(info); @@ -4493,8 +4509,12 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); + auto subPath = storePath.clone(); + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4628,11 +4648,11 @@ std::shared_ptr Worker::makeBasicDerivationGoal(StorePath && drv } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared(path.clone(), *this, repair); + goal = std::make_shared(path.clone(), *this, repair, ca); substitutionGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1b0a055d3..109e9e473 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,12 +108,12 @@ void Store::computeFSClosure(const StorePath & startPath, } -static std::optional getDerivationCA(ref drv) +std::optional getDerivationCA(const BasicDerivation & drv) { - auto outputHashMode = drv->env.find("outputHashMode"); - auto outputHashAlgo = drv->env.find("outputHashAlgo"); - auto outputHash = drv->env.find("outputHash"); - if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) { + auto outputHashMode = drv.env.find("outputHashMode"); + auto outputHashAlgo = drv.env.find("outputHashAlgo"); + auto outputHash = drv.env.find("outputHash"); + if (outputHashMode != drv.env.end() && outputHashAlgo != drv.env.end() && outputHash != drv.env.end()) { auto ht = parseHashType(outputHashAlgo->second); auto h = Hash(outputHash->second, ht); FileIngestionMethod ingestionMethod; @@ -182,7 +182,7 @@ void Store::queryMissing(const std::vector & targets, paths.insert(outPath.clone()); std::map pathsCA = {}; - if (auto ca = getDerivationCA(drv)) + if (auto ca = getDerivationCA(*drv)) pathsCA.insert({outPathS, *ca}); querySubstitutablePathInfos(paths, infos, pathsCA); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9412785b4..9918a36c6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -855,4 +855,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); +std::optional getDerivationCA(const BasicDerivation & drv); + } diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index f4b86d251..aaa38f2ed 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -32,13 +32,13 @@ cmp $outPath fetchurl.sh # Test that we can substitute from a different store dir. clearStore -other_store=file://$TEST_ROOT/other_store +other_store=file://$TEST_ROOT/other_store?store=/fnord/store hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build -vvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ From 9ac0d98a59ed042f698685234109d4b69bccf29c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 13 Jun 2020 00:14:30 -0500 Subject: [PATCH 15/70] Remove -vvvvv from tests/fetchurl.sh nix-build call --- tests/fetchurl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index aaa38f2ed..0f2044342 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -38,7 +38,7 @@ hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) -outPath=$(nix-build -vvvvv '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ From 6438ba1e990ddf76602a70d5c1143b73ed31855c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 12:31:01 -0400 Subject: [PATCH 16/70] Update strings from review comment --- src/libstore/store-api.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index bc9a6c4f5..2593b245c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -670,7 +670,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) - debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (dstStore->isValidPath(storePathForDst)) { @@ -695,7 +695,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (storePathForDst != storePath) - debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } if (!dstStore->isValidPath(storePathForDst)) { diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 26edcf95a..7abb82556 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -26,7 +26,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "flat", .shortName = 0, - .description = "use flat file ingestion", + .description = "add flat file to the Nix store", .handler = {&ingestionMethod, FileIngestionMethod::Flat}, }); } From 8974755d1958824e732640f8131b0ed22ebd703b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 14:04:46 -0400 Subject: [PATCH 17/70] Add assert for replaced storePath --- src/libstore/build.cc | 10 ++++++++-- src/libstore/local-store.cc | 2 ++ src/libstore/store-api.cc | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9753db7b1..bc67ae31e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4409,8 +4409,11 @@ void SubstitutionGoal::tryNext() subs.pop_front(); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } try { // FIXME: make async @@ -4535,8 +4538,11 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) + if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } copyStorePath(ref(sub), ref(worker.store.shared_from_this()), subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a80026258..d08c7dc6a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -853,6 +853,8 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, // recompute store path so that we can use a different store root if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { subPath = makeFixedOutputPathFromCA(path.name(), ca->second); + if (sub->storeDir == storeDir) + assert(subPath == path); if (subPath != path) debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a28895086..80a69717e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -598,6 +598,8 @@ void copyStorePath(ref srcStore, ref dstStore, if (info->isContentAddressed(*srcStore)) { auto info2 = make_ref(*info); info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(info->path == info2->path); info = info2; } @@ -670,6 +672,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto storePathForDst = storePath; if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } @@ -695,6 +699,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto storePathForDst = storePath; if (info->isContentAddressed(*srcStore)) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } From be50de1142b0600afebde32130e8561e28b63b41 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 14:14:22 -0400 Subject: [PATCH 18/70] Make sure references are empty for store path replacing also copy info2 instead of casting --- src/libstore/build.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bc67ae31e..63cb70ef2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4437,8 +4437,8 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub)) { - auto info2 = std::const_pointer_cast(std::shared_ptr(info)); + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d08c7dc6a..3230312c9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -863,7 +863,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, try { auto info = sub->queryPathInfo(subPath); - if (sub->storeDir != storeDir && !info->isContentAddressed(*sub)) + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) continue; auto narInfo = std::dynamic_pointer_cast( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80a69717e..c8097da52 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -595,7 +595,7 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) @@ -670,7 +670,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); @@ -697,7 +697,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore)) { + if (info->isContentAddressed(*srcStore) && info->references.empty()) { storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); From 5e631e3304aaf00b912c12025debfd35942a0ca9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 15:03:05 -0400 Subject: [PATCH 19/70] Add StorePathCAMap for querySubstitutablePathInfos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m not 100% sure this is wanted since it kind of makes everything have to know about ca even if they don’t really want to. But it also make things easier in dealing with looking up ca. --- src/libstore/daemon.cc | 7 +++++-- src/libstore/local-store.cc | 18 ++++++++---------- src/libstore/local-store.hh | 5 ++--- src/libstore/misc.cc | 7 ++----- src/libstore/path.hh | 2 ++ src/libstore/remote-store.cc | 14 ++++++++------ src/libstore/remote-store.hh | 4 ++-- src/libstore/store-api.hh | 4 ++-- 8 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..f7dc16948 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -612,7 +612,10 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); + StorePathCAMap pathsMap = {}; + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3230312c9..3901d532a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -841,22 +841,20 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA) +void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { for (auto & path : paths) { - auto subPath(path); + auto subPath(path.first); - auto ca = pathsCA.find(printStorePath(path)); // recompute store path so that we can use a different store root - if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) { - subPath = makeFixedOutputPathFromCA(path.name(), ca->second); + if (path.second && (hasPrefix(*path.second, "fixed:") || hasPrefix(*path.second, "text:"))) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); if (sub->storeDir == storeDir) - assert(subPath == path); - if (subPath != path) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri()); + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); } else if (sub->storeDir != storeDir) continue; debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); @@ -868,7 +866,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ + infos.insert_or_assign(path.first, SubstitutablePathInfo{ info->deriver, info->references, narInfo ? narInfo->fileSize : 0, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 183d27c80..ab9fbfbe4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -139,9 +139,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, - std::map pathsCA = {}) override; + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6902dfa0c..dc991e227 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -178,10 +178,7 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - std::map pathsCA = {}; - if (auto ca = getDerivationCA(*drv)) - pathsCA.insert({outPathS, *ca}); - querySubstitutablePathInfos({outPath}, infos, pathsCA); + querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -238,7 +235,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path.path}, infos); + querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index aaebd3ec3..91962d114 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -62,6 +62,8 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; +typedef std::map> StorePathCAMap; + /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 396629a32..4538ccd4f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -308,18 +308,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; @@ -329,12 +328,15 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); writeStorePaths(*this, conn->to, paths); conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 498a5c6a0..07ed2b51a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -55,8 +55,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA = {}) override; + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 65954574f..12a6b6939 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -445,8 +445,8 @@ public: /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA = {}) { return; }; + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, From 0c9c1b8826395a9023606cd7749aa16ef6771941 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 14:48:57 -0400 Subject: [PATCH 20/70] Return map of StorePaths in copyPaths This allows the caller to know what values were actually added to the store. --- src/libstore/store-api.cc | 12 ++++++++++-- src/libstore/store-api.hh | 17 ++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 661dea141..0458fd9a6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -638,7 +638,7 @@ void copyStorePath(ref srcStore, ref dstStore, } -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); @@ -647,7 +647,11 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st for (auto & path : storePaths) if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); - if (missing.empty()) return; + std::map pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); + + if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -677,6 +681,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } + pathsMap.insert_or_assign(storePath, storePathForDst); if (dstStore->isValidPath(storePathForDst)) { nrDone++; @@ -704,6 +709,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); } + pathsMap.insert_or_assign(storePath, storePathForDst); if (!dstStore->isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); @@ -723,6 +729,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrDone++; showProgress(); }); + + return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index fd3652588..2b12879e3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -444,8 +444,9 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; @@ -743,11 +744,13 @@ void copyStorePath(ref srcStore, ref dstStore, /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From e8e1f5282ff213be010cbe45772b9e725c01f285 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 15:19:09 -0400 Subject: [PATCH 21/70] Replace error message in getDerivationCA --- src/libstore/misc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 128c3c987..a01e81b78 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -126,7 +126,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) else if (outputHashMode->second == "flat") ingestionMethod = FileIngestionMethod::Flat; else - throw Error("unknown ingestion method: '%s'", outputHashMode->second); + throw Error("unknown outputHashMode: '%s'", outputHashMode->second); return makeFixedOutputCA(ingestionMethod, h); } From 20799a5151cbb185c1772b8e5160493b2dc2d0e8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: [PATCH 22/70] WIP: Make Hash always store a valid hash type --- src/libstore/build.cc | 2 +- src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 65 +++++++++++++----------- src/libutil/hash.hh | 17 +++---- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix-store/nix-store.cc | 10 ++-- src/nix/add-to-store.cc | 4 +- src/nix/develop.cc | 2 +- 8 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 82a2ab831..86041cabd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4997,7 +4997,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a05048290..de9b6a791 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -116,7 +116,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e49eb4569..00bb999cb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,16 +16,19 @@ namespace nix { +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; + } + abort(); +} void Hash::init() { - if (!type) abort(); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; - } + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -105,18 +108,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) { - if (h.type) - return *h.type; - else - abort(); -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -137,29 +133,36 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional type) - : type(type) +Hash::Hash(std::string_view s, std::optional optType) { size_t pos = 0; bool isSRI = false; + // Find the : or - separater, and set `isSRI` to the correct value auto sep = s.find(':'); if (sep == string::npos) { sep = s.find('-'); - if (sep != string::npos) { + if (sep != string::npos) isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); } + // Parse the has type before the separater, if there was one. + std::optional optParsedType; if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) + auto hts = s.substr(0, sep); + auto optParsedType = parseHashType(hts); + if (!optParsedType) throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); - pos = sep + 1; + } + + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) { + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + } else { + this->type = optParsedType ? *optParsedType : *optType; + if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); } init(); @@ -214,7 +217,7 @@ Hash::Hash(std::string_view s, std::optional type) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) @@ -267,7 +270,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -334,7 +337,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -342,7 +345,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +354,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..b51850ccf 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -25,14 +25,11 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - std::optional type = {}; - - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ Hash(HashType type) : type(type) { init(); }; @@ -105,7 +102,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); @@ -121,9 +118,9 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..22410c44c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,7 +153,7 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + Hash hash(ht), expectedHash(ht); std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4e02aa2bf..d061317ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash && info->narHash->type == htSHA256); + cout << fmt("%s\n", info->narHash->to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash->type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash->to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..0dda2af38 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,9 +46,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + *info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8b85caf82..4aee9f202 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -139,7 +139,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) .path = shellOutPath, .hash = DerivationOutputHash { .method = FileIngestionMethod::Flat, - .hash = Hash { }, + .hash = Hash { htSHA256 }, }, }); drv.env["out"] = store->printStorePath(shellOutPath); From e7a14118df5b53b07ecf48c8fb1ac712677250b3 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 19 Jun 2020 16:50:28 -0400 Subject: [PATCH 23/70] WIP bug fixing --- src/libexpr/primops.cc | 8 ++++---- src/libexpr/primops/fetchTree.cc | 2 +- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/build.cc | 10 ++++++---- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/derivations.cc | 2 +- src/libstore/export-import.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/nar-info.hh | 2 +- src/libstore/references.cc | 6 +++--- src/libstore/references.hh | 3 +-- src/nix/make-content-addressable.cc | 2 +- src/nix/verify.cc | 8 ++++---- 14 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f5fbd3fa6..0dd5624b2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1122,7 +1122,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1153,7 +1153,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1187,7 +1187,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .nixCode = NixCode { .errPos = pos } }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1197,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + Hash expectedHash(htSHA256); for (auto & attr : *args[0]->attrs) { const string & n(attr.name); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710a..a5b836383 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -147,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..98a3eaebb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 86041cabd..80351a675 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3730,8 +3730,8 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); @@ -3777,8 +3777,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..f3827684b 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..296bfad5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -646,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..53bd4da2a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,7 +9,7 @@ namespace nix { std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..9b8cc5c3a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..171903980 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash(s); conn->from >> info->ca; info->sigs = readStrings(conn->from); } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..8ee8d1ae8 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -79,8 +79,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { RefScanSink sink; std::map backMap; @@ -114,7 +114,7 @@ PathSet scanForReferences(const string & path, hash = sink.hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..a712dceef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index d1aba08e3..c92f894f2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -88,15 +88,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (info->ca == "") - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != info->narHash) { + if (hash.first != *info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -104,7 +104,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), hash.first.to_string(Base32, true)) }); } From 507aa48739f23f9a16c8b7079bfa6fc1806be78e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: [PATCH 24/70] WIP: Make Hash always store a valid hash type --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 8 +-- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/tree-info.cc | 2 +- src/libfetchers/tree-info.hh | 2 +- src/libstore/binary-cache-store.cc | 6 +-- src/libstore/build.cc | 12 +++-- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/daemon.cc | 4 +- src/libstore/derivations.cc | 2 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/local-store.cc | 26 ++++----- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 27 +++++----- src/libstore/nar-info.hh | 2 +- src/libstore/references.cc | 8 +-- src/libstore/references.hh | 3 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 68 +++++++++++++----------- src/libutil/hash.hh | 17 +++--- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix-store/nix-store.cc | 10 ++-- src/nix/add-to-store.cc | 4 +- src/nix/develop.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- src/nix/verify.cc | 8 +-- 29 files changed, 126 insertions(+), 124 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7..2a2a0d429 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash->to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f5fbd3fa6..0dd5624b2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1122,7 +1122,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1153,7 +1153,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1187,7 +1187,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .nixCode = NixCode { .errPos = pos } }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1197,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + Hash expectedHash(htSHA256); for (auto & attr : *args[0]->attrs) { const string & n(attr.name); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710a..d23d47592 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -23,7 +23,7 @@ void emitTreeAttrs( assert(tree.info.narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI, true)); + tree.info.narHash->to_string(SRI, true)); if (input->getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); @@ -147,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index b2d8cfc8d..432aa6182 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,7 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); } } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c7347281..9d1872097 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -11,7 +11,7 @@ namespace nix::fetchers { struct TreeInfo { - Hash narHash; + std::optional narHash; std::optional revCount; std::optional lastModified; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..98a3eaebb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 82a2ab831..80351a675 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3730,8 +3730,8 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); @@ -3777,8 +3777,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; @@ -4997,7 +4999,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..f3827684b 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..296bfad5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -646,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..53bd4da2a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,7 +9,7 @@ namespace nix { std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..9b8cc5c3a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..d3880459c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash(s); conn->from >> info->ca; info->sigs = readStrings(conn->from); } @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6b55ff7c..aa2ee5b30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -897,7 +897,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash && i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1010,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1067,12 +1067,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + HashResult hash = method == FileIngestionMethod::Recursive + ? HashResult { + hashAlgo == htSHA256 ? h : hashString(htSHA256, dump), + dump.size(), + } + : hashPath(htSHA256, realPath); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1255,9 +1255,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr hashSink; if (info->ca == "" || !info->references.count(info->path)) - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1266,7 +1266,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 552970248..6036b905e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash->to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index bb4448c90..fb84b0410 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { return Hash(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } else if (name == "NarHash") narHash = parseHashField(value); else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); + if (!ca.empty()) throw corrupt(); ca = value; } @@ -76,7 +75,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -86,11 +85,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash && narHash->type == htSHA256); + res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..4733bc388 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -79,8 +79,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { RefScanSink sink; std::map backMap; @@ -112,9 +112,9 @@ PathSet scanForReferences(const string & path, found.insert(j->second); } - hash = sink.hashSink.finish(); + auto hash = sink.hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f5f2ab7fd..c206a4538 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -462,7 +462,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 982fc22b6..c74aeec48 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -430,7 +430,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash->to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -462,7 +462,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narHash", info->narHash->to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -505,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -746,7 +746,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash->to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a05048290..de9b6a791 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -116,7 +116,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e49eb4569..35a449530 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,16 +16,19 @@ namespace nix { +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; + } + abort(); +} void Hash::init() { - if (!type) abort(); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; - } + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -105,18 +108,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) { - if (h.type) - return *h.type; - else - abort(); -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -137,31 +133,39 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional type) - : type(type) +Hash::Hash(std::string_view s, std::optional optType) { size_t pos = 0; bool isSRI = false; + // Find the : or - separater, and set `isSRI` to the correct value auto sep = s.find(':'); - if (sep == string::npos) { + if (sep == std::string_view::npos) { sep = s.find('-'); - if (sep != string::npos) { + if (sep != std::string_view::npos) isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); } - if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + if (sep != std::string_view::npos) { + auto hts = s.substr(0, sep); + auto optParsedType = parseHashTypeOpt(hts); + if (!optParsedType) throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); pos = sep + 1; } + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) { + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + } else { + this->type = optParsedType ? *optParsedType : *optType; + if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); + } + init(); size_t size = s.size() - pos; @@ -214,7 +218,7 @@ Hash::Hash(std::string_view s, std::optional type) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) @@ -267,7 +271,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -334,7 +338,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -342,7 +346,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +355,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..b51850ccf 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -25,14 +25,11 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - std::optional type = {}; - - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ Hash(HashType type) : type(type) { init(); }; @@ -105,7 +102,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); @@ -121,9 +118,9 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..22410c44c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,7 +153,7 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + Hash hash(ht), expectedHash(ht); std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4e02aa2bf..d061317ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash && info->narHash->type == htSHA256); + cout << fmt("%s\n", info->narHash->to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash->type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash->to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f43f774c1..0dda2af38 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,9 +46,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + *info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8b85caf82..4aee9f202 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -139,7 +139,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) .path = shellOutPath, .hash = DerivationOutputHash { .method = FileIngestionMethod::Flat, - .hash = Hash { }, + .hash = Hash { htSHA256 }, }, }); drv.env["out"] = store->printStorePath(shellOutPath); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..a712dceef 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, *info.narHash); if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index d1aba08e3..c92f894f2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -88,15 +88,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (info->ca == "") - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != info->narHash) { + if (hash.first != *info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -104,7 +104,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), hash.first.to_string(Base32, true)) }); } From f2a6cee334255ced72a70f527cf3c283b4586e42 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 18:06:19 -0400 Subject: [PATCH 25/70] Update worker protocol to support sending storepath maps We need to also send the ca to daemon in addition to the path. --- src/libstore/daemon.cc | 11 +++++++---- src/libstore/remote-store.cc | 32 ++++++++++++++++++++++++++++---- src/libstore/worker-protocol.hh | 6 +++++- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f7dc16948..c613ab47d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -609,12 +609,15 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePathInfos: { - auto paths = readStorePaths(*store, from); - logger->startWork(); SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); + if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + auto paths = readStorePaths(*store, from); + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + } else + pathsMap = readStorePathCAMap(*store, from); + logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4538ccd4f..f6158685c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,6 +38,27 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +std::map> readStorePathCAMap(const Store & store, Source & from) +{ + StorePathCAMap paths; + auto count = readNum(from); + while (count--) { + auto path = readString(from); + auto ca = readString(from); + paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt); + } + return paths; +} + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +{ + out << paths.size(); + for (auto & i : paths) { + out << store.printStorePath(i.first); + out << (i.second ? *i.second : ""); + } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -334,10 +355,13 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << wopQuerySubstitutablePathInfos; - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + writeStorePaths(*this, conn->to, paths); + } else + writeStorePathCAMap(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ac42457fc..d2944674f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x115 +#define PROTOCOL_VERSION 0x116 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -69,5 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); +std::map> readStorePathCAMap(const Store & store, Source & from); + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); + } From 73ac003b377aa4750de057bba2de949295524a9c Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 22 Jun 2020 10:03:19 -0400 Subject: [PATCH 26/70] More bug fixing --- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 171903980..d3880459c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6b55ff7c..e276576e1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -897,7 +897,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1010,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", From ec3a85730742497b29782c0b545911eb78d3c65e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jun 2020 18:19:40 +0000 Subject: [PATCH 27/70] Fix and clean up hash parser --- src/libutil/hash.cc | 62 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index f44210737..70a18f9a0 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -133,68 +133,66 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional optType) +Hash::Hash(std::string_view original, std::optional optType) { + auto rest = original; + size_t pos = 0; bool isSRI = false; - // Find the : or - separater, and set `isSRI` to the correct value - auto sep = s.find(':'); - if (sep == std::string_view::npos) { - sep = s.find('-'); - if (sep != std::string_view::npos) - isSRI = true; - } - // Parse the has type before the separater, if there was one. std::optional optParsedType; - if (sep != std::string_view::npos) { - auto hts = s.substr(0, sep); - auto optParsedType = parseHashTypeOpt(hts); - if (!optParsedType) - throw BadHash("unknown hash type '%s'", hts); - pos = sep + 1; + { + auto sep = rest.find(':'); + if (sep == std::string_view::npos) { + sep = rest.find('-'); + if (sep != std::string_view::npos) + isSRI = true; + } + if (sep != std::string_view::npos) { + auto hashRaw = rest.substr(0, sep); + optParsedType = parseHashType(hashRaw); + rest = rest.substr(sep + 1); + } } // Either the string or user must provide the type, if they both do they // must agree. if (!optParsedType && !optType) { - throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", s); + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); } else { this->type = optParsedType ? *optParsedType : *optType; if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*optType)); + throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); } init(); - size_t size = s.size() - pos; - - if (!isSRI && size == base16Len()) { + if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", s); + throw BadHash("invalid base-16 hash '%s'", original); }; for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(s[pos + i * 2]) << 4 - | parseHexDigit(s[pos + i * 2 + 1]); + parseHexDigit(rest[pos + i * 2]) << 4 + | parseHexDigit(rest[pos + i * 2 + 1]); } } - else if (!isSRI && size == base32Len()) { + else if (!isSRI && rest.size() == base32Len()) { - for (unsigned int n = 0; n < size; ++n) { - char c = s[pos + size - n - 1]; + for (unsigned int n = 0; n < rest.size(); ++n) { + char c = rest[rest.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -204,21 +202,21 @@ Hash::Hash(std::string_view s, std::optional optType) hash[i + 1] |= digit >> (8 - j); } else { if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); } } } - else if (isSRI || size == base64Len()) { - auto d = base64Decode(s.substr(pos)); + else if (isSRI || rest.size() == base64Len()) { + auto d = base64Decode(rest); if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); assert(hashSize); memcpy(hash, d.data(), hashSize); } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) From 8d51d38e4ccf2ca0fa0b0471b32531fe287e38a4 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 23 Jun 2020 17:16:20 -0400 Subject: [PATCH 28/70] Fix test suite --- src/nix/add-to-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index de68e9501..ad1f9e91f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -46,7 +46,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - *info.narHash = narHash; + info.narHash = narHash; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { .method = FileIngestionMethod::Recursive, From 1f9cb06db23f357f06404969820be1d3d4118a18 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 2 Jul 2020 10:59:24 -0400 Subject: [PATCH 29/70] Try next when no ca exists and have different store dirs --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 95280ef1e..20d3c7387 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4421,6 +4421,9 @@ void SubstitutionGoal::tryNext() subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + tryNext(); + return; } try { From d2e8b9ff0e8048d7b3914811b2d4bd535feac49d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 2 Jul 2020 11:12:05 -0400 Subject: [PATCH 30/70] Store subPath in SubstitutionGoal --- src/libstore/build.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 20d3c7387..12e4b6c3b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4270,6 +4270,10 @@ private: /* The store path that should be realised through a substitute. */ StorePath storePath; + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; + /* The remaining substituters. */ std::list> subs; @@ -4416,7 +4420,6 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - auto subPath = storePath; if (ca) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) @@ -4428,7 +4431,7 @@ void SubstitutionGoal::tryNext() try { // FIXME: make async - info = sub->queryPathInfo(subPath); + info = sub->queryPathInfo(subPath ? *subPath : storePath); } catch (InvalidPath &) { tryNext(); return; @@ -4548,15 +4551,8 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - auto subPath = storePath; - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } - copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { From 1be279af2622b196cc4630c48254adc96071c7e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Jul 2020 21:44:18 +0000 Subject: [PATCH 31/70] Fix Narinfo corruption detection bug The aim of this check was just to ensure each key occurs once. --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index c403d4bec..ca471463c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -66,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!value.empty()) throw corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } From fd42176a21d62039580c6cf6430ee19d8d3b6e1c Mon Sep 17 00:00:00 2001 From: Daniel Fitzpatrick Date: Mon, 6 Jul 2020 21:59:18 -0500 Subject: [PATCH 32/70] Add a script to install nix on non-systemd systems. --- scripts/install-generic-multi-user.sh | 799 ++++++++++++++++++++++++++ scripts/install-nix-from-closure.sh | 14 +- 2 files changed, 809 insertions(+), 4 deletions(-) create mode 100644 scripts/install-generic-multi-user.sh diff --git a/scripts/install-generic-multi-user.sh b/scripts/install-generic-multi-user.sh new file mode 100644 index 000000000..4353e9a06 --- /dev/null +++ b/scripts/install-generic-multi-user.sh @@ -0,0 +1,799 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +# installer allows overriding build user count to speed up installation +# as creating each user takes non-trivial amount of time on macos +readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" +readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" +readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + step=$((step + 1)) + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat < $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure < /dev/null >&2; then + failure < "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + fi +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat < "$SCRATCH/nix.conf" +$NIX_EXTRA_CONF +build-users-group = $NIX_BUILD_GROUP_NAME +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # A lot of this is still usable for a non-systemd multi-user install + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then + validate_starting_assumptions + fi + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + + trap finish_success EXIT +} + + +main diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 5824c2217..906e7da9b 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -35,7 +35,7 @@ fi # Determine if we could use the multi-user installer or not if [ "$(uname -s)" = "Darwin" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 -elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then +elif [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi @@ -122,9 +122,15 @@ if [ "$(uname -s)" = "Darwin" ]; then fi if [ "$INSTALL_MODE" = "daemon" ]; then - printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' - exec "$self/install-multi-user" - exit 0 + if [ -e /run/systemd/system ]; then + printf '\e[1;31mSwitching to the Systemd Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" + exit 0 + else + printf '\e[1;31mSwitching to the Non-Systemd Daemon-based Installer\e[0m\n' + exec "$self/install-generic-multi-user" + exit 0 + fi fi if [ "$(id -u)" -eq 0 ]; then From 39859b853cef3f522c2b396cfd47f4fe184e1781 Mon Sep 17 00:00:00 2001 From: Daniel Fitzpatrick Date: Tue, 7 Jul 2020 15:08:23 -0500 Subject: [PATCH 33/70] merged systemd installer with non-systemd installer. --- scripts/install-generic-multi-user.sh | 799 -------------------------- scripts/install-multi-user.sh | 43 +- scripts/install-nix-from-closure.sh | 12 +- 3 files changed, 37 insertions(+), 817 deletions(-) delete mode 100644 scripts/install-generic-multi-user.sh diff --git a/scripts/install-generic-multi-user.sh b/scripts/install-generic-multi-user.sh deleted file mode 100644 index 4353e9a06..000000000 --- a/scripts/install-generic-multi-user.sh +++ /dev/null @@ -1,799 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - -# Sourced from: -# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh -# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 -# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh -# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh -# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a -# -# however tracking which bits came from which would be impossible. - -readonly ESC='\033[0m' -readonly BOLD='\033[1m' -readonly BLUE='\033[34m' -readonly BLUE_UL='\033[4;34m' -readonly GREEN='\033[32m' -readonly GREEN_UL='\033[4;32m' -readonly RED='\033[31m' - -# installer allows overriding build user count to speed up installation -# as creating each user takes non-trivial amount of time on macos -readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} -readonly NIX_BUILD_GROUP_ID="30000" -readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" -# Please don't change this. We don't support it, because the -# default shell profile that comes with Nix doesn't support it. -readonly NIX_ROOT="/nix" -readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} - -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") -readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" -readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" - -readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" -readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" -readonly EXTRACTED_NIX_PATH="$(dirname "$0")" - -readonly ROOT_HOME=$(echo ~root) - -if [ -t 0 ]; then - readonly IS_HEADLESS='no' -else - readonly IS_HEADLESS='yes' -fi - -headless() { - if [ "$IS_HEADLESS" = "yes" ]; then - return 0 - else - return 1 - fi -} - -contactme() { - echo "We'd love to help if you need it." - echo "" - echo "If you can, open an issue at https://github.com/nixos/nix/issues" - echo "" - echo "Or feel free to contact the team," - echo " - on IRC #nixos on irc.freenode.net" - echo " - on twitter @nixos_org" -} - -uninstall_directions() { - subheader "Uninstalling nix:" - local step=0 - - step=$((step + 1)) - - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then - step=$((step + 1)) - cat < $1" -} - -bold() { - echo "$BOLD$*$ESC" -} - -ok() { - _textout "$GREEN" "$@" -} - -warning() { - warningheader "warning!" - cat - echo "" -} - -failure() { - header "oh no!" - _textout "$RED" "$@" - echo "" - _textout "$RED" "$(contactme)" - trap finish_cleanup EXIT - exit 1 -} - -ui_confirm() { - _textout "$GREEN$GREEN_UL" "$1" - - if headless; then - echo "No TTY, assuming you would say yes :)" - return 0 - fi - - local prompt="[y/n] " - echo -n "$prompt" - while read -r y; do - if [ "$y" = "y" ]; then - echo "" - return 0 - elif [ "$y" = "n" ]; then - echo "" - return 1 - else - _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" - echo -n "$prompt" - fi - done - echo "" - return 1 -} - -__sudo() { - local expl="$1" - local cmd="$2" - shift - header "sudo execution" - - echo "I am executing:" - echo "" - printf " $ sudo %s\\n" "$cmd" - echo "" - echo "$expl" - echo "" - - return 0 -} - -_sudo() { - local expl="$1" - shift - if ! headless; then - __sudo "$expl" "$*" - fi - sudo "$@" -} - - -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { - rm -rf "$SCRATCH" -} - -function finish_fail { - finish_cleanup - - failure < /dev/null >&2; then - failure < "$SCRATCH/.nix-channels" - _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" - fi -} - -welcome_to_nix() { - ok "Welcome to the Multi-User Nix Installation" - - cat < "$SCRATCH/nix.conf" -$NIX_EXTRA_CONF -build-users-group = $NIX_BUILD_GROUP_NAME -EOF - _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf -} - -main() { - if [ "$(uname -s)" = "Darwin" ]; then - # shellcheck source=./install-darwin-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # A lot of this is still usable for a non-systemd multi-user install - else - failure "Sorry, I don't know what to do on $(uname)" - fi - - welcome_to_nix - chat_about_sudo - - if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then - validate_starting_assumptions - fi - - setup_report - - if ! ui_confirm "Ready to continue?"; then - ok "Alright, no changes have been made :)" - contactme - trap finish_cleanup EXIT - exit 1 - fi - - create_build_group - create_build_users - create_directories - place_channel_configuration - install_from_extracted_nix - - configure_shell_profile - - set +eu - . /etc/profile - set -eu - - setup_default_profile - place_nix_configuration - - trap finish_success EXIT -} - - -main diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 00c9d540b..e5cc4d7ed 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix- readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" +#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" +#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" readonly EXTRACTED_NIX_PATH="$(dirname "$0")" readonly ROOT_HOME=$(echo ~root) @@ -69,9 +71,11 @@ uninstall_directions() { subheader "Uninstalling nix:" local step=0 - if poly_service_installed_check; then + if [ -e /run/systemd/system ] && poly_service_installed_check; then step=$((step + 1)) poly_service_uninstall_directions "$step" + else + step=$((step + 1)) fi for profile_target in "${PROFILE_TARGETS[@]}"; do @@ -250,7 +254,9 @@ function finish_success { echo "But fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." fi - cat < Date: Wed, 8 Jul 2020 15:38:01 -0400 Subject: [PATCH 34/70] Add name to BasicDerivation We always have a name for BasicDerivation, since we have a derivation store path that has a name. --- src/libexpr/primops.cc | 6 ++++-- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 16 ++++++++++------ src/libstore/derivations.hh | 5 +++-- src/nix-store/nix-store.cc | 4 ++-- src/nix/repl.cc | 2 +- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dec917b38..da1937e6b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -102,9 +102,11 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + StorePath storePath = state.store->parseStorePath(path); + // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { - Derivation drv = readDerivation(*state.store, realPath); + if (state.store->isStorePath(path) && state.store->isValidPath(storePath) && isDerivation(path)) { + Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ebc4d0285..4cf67064a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -474,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, std::string(drvPath.name())); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..bdc6f4f68 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -129,9 +129,11 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } -static Derivation parseDerivation(const Store & store, const string & s) +static Derivation parseDerivation(const Store & store, const string & s, string name) { Derivation drv; + drv.name = name; + istringstream_nocopy str(s); expect(str, "Derive(["); @@ -175,10 +177,10 @@ static Derivation parseDerivation(const Store & store, const string & s) } -Derivation readDerivation(const Store & store, const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath, std::string name) { try { - return parseDerivation(store, readFile(drvPath)); + return parseDerivation(store, readFile(drvPath), name); } catch (FormatError & e) { throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } @@ -196,7 +198,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), std::string(drvPath.name())); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -369,7 +371,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } @@ -435,8 +437,10 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum(in); for (size_t n = 0; n < nr; n++) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..f9d328196 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -35,6 +35,7 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + string name; BasicDerivation() { } virtual ~BasicDerivation() { }; @@ -76,7 +77,7 @@ StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath, string name); // FIXME: remove bool isDerivation(const string & fileName); @@ -93,7 +94,7 @@ bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..9a8cf0f2f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // informational only + auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv); + readDerivation(in, *store, drv, std::string(drvPath.name())); getBuildSettings(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b..a3d85ed31 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -483,7 +483,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath); + auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); From 8e0d0689be797f9e42f9b43b06f50c1af7f20b4a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 8 Jul 2020 19:11:39 -0400 Subject: [PATCH 35/70] Only store hash of fixed derivation output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we don’t need a full storepath for a fixedoutput derivation. So just putting the ingestion method + the hash is sufficient. --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/get-drvs.cc | 2 +- src/libexpr/primops.cc | 27 ++++---- src/libstore/build.cc | 76 ++++++++++++----------- src/libstore/builtins/fetchurl.cc | 21 ++++--- src/libstore/derivations.cc | 100 +++++++++++++++++++----------- src/libstore/derivations.hh | 18 ++++-- src/libstore/local-store.cc | 11 +--- src/libstore/misc.cc | 4 +- src/nix-env/nix-env.cc | 2 +- src/nix-store/nix-store.cc | 6 +- src/nix/develop.cc | 4 +- src/nix/repl.cc | 2 +- src/nix/show-derivation.cc | 8 +-- 14 files changed, 162 insertions(+), 121 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7..6dba07549 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -304,7 +304,7 @@ SV * derivationFromPath(char * drvPath) HV * outputs = newHV(); for (auto & i : drv.outputs) - hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0); + hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path(store(), drv.name)).c_str(), 0), 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 9055f59a1..5d6e39aa0 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); - outPath = store->printStorePath(i->second.path); + outPath = store->printStorePath(i->second.path(*store, drv.name)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index da1937e6b..9b613eef9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -64,7 +64,7 @@ void EvalState::realiseContext(const PathSet & context) DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); - allowedPaths->insert(store->printStorePath(i->second.path)); + allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name))); } } } @@ -120,7 +120,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -778,11 +778,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .path = std::move(outPath), - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, + }, }); } @@ -797,8 +798,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .path = StorePath::dummy, - .hash = std::optional {}, + .output = DerivationOutputIntensional { + .path = StorePath::dummy, + }, }); } @@ -809,8 +811,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .path = std::move(outPath), - .hash = std::optional(), + .output = DerivationOutputIntensional { + .path = std::move(outPath), + }, }); } } @@ -831,7 +834,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); + state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0ef2f288f..bf7d4c2f9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store))); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation() retrySubstitution = false; for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + worker.store.addTempRoot(i.second.path(worker.store, drv->name)); /* Check what outputs paths are not already valid. */ auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); @@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure() StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure); } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + outputClosure.erase(i.second.path(worker.store, drv->name)); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i); } /* Check each path (slow!). */ @@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised() for (auto & j : i.second) { auto k = inDrv.outputs.find(j); if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); + worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths); else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", @@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild() goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) + for (auto & outPath : drv->outputPaths(worker.store)) lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { @@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = drv->outputPaths(); + missingPaths = drv->outputPaths(worker.store); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name))); + deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name))); } /* Don't do a remote build if the derivation has the attribute @@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + auto outputPaths = drv->outputPaths(worker.store); std::map hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths); } } @@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder() /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name)); /* Construct the environment passed to the builder. */ initEnv(); @@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2613,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2818,7 +2818,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); + newPaths.insert(output.second.path(*this, drv.name)); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -3580,7 +3580,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); + result.insert(drv.outputs.find(i)->second.path(store, drv.name)); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3618,7 +3618,7 @@ void DerivationGoal::registerOutputs() if (hook) { bool allValid = true; for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false; if (allValid) return; } @@ -3639,23 +3639,23 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name)); for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); + if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue; Path actualPath = path; if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); + auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name)); if (r != redirectedOutputs.end()) { auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) + && redirectedBadOutputs.count(i.second.path(worker.store, drv->name)) && pathExists(redirected)) replaceValidPath(path, redirected); if (buildMode == bmCheck) @@ -3724,7 +3724,9 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - if (i.second.hash->method == FileIngestionMethod::Flat) { + FixedOutputHash outputHash = std::get(i.second.output).hash; + + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3735,13 +3737,13 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + Hash h2 = outputHash.method == FileIngestionMethod::Recursive + ? hashPath(*outputHash.hash.type, actualPath).first + : hashFile(*outputHash.hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); - if (i.second.hash->hash != h2) { + if (outputHash.hash != h2) { /* Throw an error after registering the path as valid. */ @@ -3749,7 +3751,7 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), + outputHash.hash.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3771,7 +3773,7 @@ void DerivationGoal::registerOutputs() assert(worker.store.parseStorePath(path) == dest); ca = FixedOutputHash { - .method = i.second.hash->method, + .method = outputHash.method, .hash = h2, }; } @@ -3895,7 +3897,7 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3913,7 +3915,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix; deletePath(prev); } } @@ -4214,9 +4216,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + worker.store.isValidPath(i.second.path(worker.store, drv->name)) && + (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name))); + if (good == returnValid) result.insert(i.second.path(worker.store, drv->name)); } return result; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..017a633b5 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,16 +63,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto & output = drv.outputs.begin()->second; /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); + if (auto hash = std::get_if(&output.output)) { + if (hash->hash.method == FileIngestionMethod::Flat) { + for (auto hashedMirror : settings.hashedMirrors.get()) { + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + fetch(hashedMirror + printHashType(*hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } } + } + } /* Otherwise try the specified URL. */ fetch(mainUrl); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bdc6f4f68..ffeafea87 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,12 +8,28 @@ namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +// FIXME Put this somewhere? +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + +StorePath DerivationOutput::path(const Store & store, string drvName) const +{ + return std::visit(overloaded { + [](DerivationOutputIntensional doi) { + return doi.path; + }, + [&](DerivationOutputFixed dof) { + return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + } + }, output); +} + +const StorePath & BasicDerivation::findOutput(const Store & store, const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) throw Error("derivation has no output '%s'", id); - return i->second.path; + return i->second.path(store, name); } @@ -108,7 +124,6 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -116,16 +131,21 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputIntensional { + .path = std::move(path), + } + }; } @@ -266,10 +286,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); + if (auto hash = std::get_if(&i.second.output)) { + s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); + s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false)); + } else { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } s += ')'; } @@ -325,7 +349,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::holds_alternative(outputs.begin()->second.output); } @@ -357,10 +381,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); + auto hash = std::get(i->second.output); return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + + hash.hash.printMethodAlgo() + ":" + + hash.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(i->second.path(store, drv.name))); } /* For other derivations, replace the inputs paths with recursive @@ -394,11 +419,11 @@ bool wantOutput(const string & output, const std::set & wanted) } -StorePathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths(const Store & store) const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path(store, name)); return paths; } @@ -408,7 +433,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); auto hash = readString(in); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -416,16 +440,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputIntensional { + .path = std::move(path), + } }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; } StringSet BasicDerivation::outputNames() const @@ -469,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << drv.outputs.size(); for (auto & i : drv.outputs) { out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); + << store.printStorePath(i.second.path(store, drv.name)); + if (auto hash = std::get_if(&i.second.output)) { + out << hash->hash.printMethodAlgo() + << hash->hash.hash.to_string(Base16, false); } else { out << "" << ""; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f9d328196..d8cee0f34 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,10 +13,20 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +struct DerivationOutputIntensional { StorePath path; - std::optional hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutputFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutput +{ + std::variant output; + StorePath path(const Store & store, string drvName) const; }; typedef std::map DerivationOutputs; @@ -42,7 +52,7 @@ struct BasicDerivation /* Return the path corresponding to the output identifier `id' in the given derivation. */ - const StorePath & findOutput(const std::string & id) const; + const StorePath & findOutput(const Store & store, const std::string & id) const; bool isBuiltin() const; @@ -50,7 +60,7 @@ struct BasicDerivation bool isFixedOutput() const; /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + StorePathSet outputPaths(const Store & store) const; /* Return the output names of a derivation. */ StringSet outputNames() const; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eed225349..7ecaa8743 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); } else { Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); + check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); } } @@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (printStorePath(i.second.path)) + (printStorePath(i.second.path(*this, drv.name))) .exec(); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..c4d22a634 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector & targets, PathSet invalid; for (auto & j : drv->outputs) if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); + && !isValidPath(j.second.path(*this, drv->name))) + invalid.insert(printStorePath(j.second.path(*this, drv->name))); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d74..a42859fd0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -381,7 +381,7 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); + elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput(*state.store, "out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9a8cf0f2f..2a2f16ac1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", store2->printStorePath(path.path), j); - auto outPath = store2->printStorePath(i->second.path); + auto outPath = store2->printStorePath(i->second.path(*store, drv.name)); if (store2) { if (gcRoot == "") printGCWarning(); @@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, auto drv = store->derivationFromPath(storePath); StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path); + outputs.insert(i.second.path(*store, drv.name)); return outputs; } else return {storePath}; @@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) - cout << fmt("%1%\n", store->printStorePath(j.second.path)); + cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name))); } break; } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index eb93f56fc..929fae5c4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,9 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputIntensional { + .path = shellOutPath + } }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a3d85ed31..bd72a3b19 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -486,7 +486,7 @@ bool NixRepl::processLine(string line) auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); } } else if (command == ":i") { runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 5d77cfdca..a868023d4 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand auto outputsObj(drvObj.object("outputs")); for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash) { - outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); - outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); + outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); + if (auto hash = std::get_if(&output.second.output)) { + outputObj.attr("hashAlgo", hash->hash.printMethodAlgo()); + outputObj.attr("hash", hash->hash.hash.to_string(Base16, false)); } } } From 06a4e15478920bcf8d39c36abc1a73a1e83bc9b7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 8 Jul 2020 19:27:51 -0400 Subject: [PATCH 36/70] Fix build.cc on linux --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bf7d4c2f9..9a53d9df7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path(drv->name))); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) From a7884970c57f821e274fba20473d6fcce95ff6c8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 9 Jul 2020 11:37:18 -0400 Subject: [PATCH 37/70] Fix DerivationOutputExtensional name --- src/libexpr/primops.cc | 4 ++-- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 4 ++-- src/nix/develop.cc | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9b613eef9..bd7ec628c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -798,7 +798,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = StorePath::dummy, }, }); @@ -811,7 +811,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(outPath), }, }); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ffeafea87..86f55342d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,7 +15,7 @@ template overloaded(Ts...) -> overloaded; StorePath DerivationOutput::path(const Store & store, string drvName) const { return std::visit(overloaded { - [](DerivationOutputIntensional doi) { + [](DerivationOutputExtensional doi) { return doi.path; }, [&](DerivationOutputFixed dof) { @@ -142,7 +142,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream }; } else return DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(path), } }; @@ -450,7 +450,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) }; } else return DerivationOutput { - .output = DerivationOutputIntensional { + .output = DerivationOutputExtensional { .path = std::move(path), } }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d8cee0f34..2515188ec 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,7 +13,7 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutputIntensional +struct DerivationOutputExtensional { StorePath path; }; @@ -25,7 +25,7 @@ struct DerivationOutputFixed struct DerivationOutput { - std::variant output; + std::variant output; StorePath path(const Store & store, string drvName) const; }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 929fae5c4..e03d2e057 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputIntensional { + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputExtensional { .path = shellOutPath } }); drv.env["out"] = store->printStorePath(shellOutPath); From abea26a9683d58e0bee43d9ededeb710f3617603 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 01:57:06 +0000 Subject: [PATCH 38/70] BasicDerivation::findOutput cannot return reference anymore --- src/libstore/derivations.cc | 2 +- src/libstore/derivations.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 86f55342d..bde147502 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -24,7 +24,7 @@ StorePath DerivationOutput::path(const Store & store, string drvName) const }, output); } -const StorePath & BasicDerivation::findOutput(const Store & store, const string & id) const +const StorePath BasicDerivation::findOutput(const Store & store, const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2515188ec..4694ec0dd 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -52,7 +52,7 @@ struct BasicDerivation /* Return the path corresponding to the output identifier `id' in the given derivation. */ - const StorePath & findOutput(const Store & store, const std::string & id) const; + const StorePath findOutput(const Store & store, const std::string & id) const; bool isBuiltin() const; From 1c9bec226f9ff1a9586bfefcd0c3b66ed989b2d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 02:38:03 +0000 Subject: [PATCH 39/70] Don't improperly assume path is store path --- src/libexpr/primops.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bd7ec628c..4367f9b22 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -102,10 +102,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - StorePath storePath = state.store->parseStorePath(path); - // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(storePath) && isDerivation(path)) { + auto isValidDerivationInStore = [&]() -> std::optional { + if (!state.store->isStorePath(path)) + return std::nullopt; + auto storePath = state.store->parseStorePath(path); + if (!(state.store->isValidPath(storePath) && isDerivation(path))) + return std::nullopt; + return storePath; + }; + if (auto optStorePath = isValidDerivationInStore()) { + auto storePath = *optStorePath; Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); From 13ec627e0a46d26b5de227e193a05b51af7f55e5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 03:03:12 +0000 Subject: [PATCH 40/70] Set derivation name in dervationStrict --- src/libexpr/primops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4367f9b22..14a7deed5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -591,6 +591,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Build the derivation expression by processing the attributes. */ Derivation drv; + drv.name = drvName; PathSet context; From 5d0b75e5b6a1c70203257fecc25743e5a1e009cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:02:36 +0000 Subject: [PATCH 41/70] std::string_view for new derivation name parameters --- src/libstore/derivations.cc | 8 ++++---- src/libstore/derivations.hh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bde147502..97fd21885 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -12,7 +12,7 @@ namespace nix { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath DerivationOutput::path(const Store & store, string drvName) const +StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { return std::visit(overloaded { [](DerivationOutputExtensional doi) { @@ -149,7 +149,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } -static Derivation parseDerivation(const Store & store, const string & s, string name) +static Derivation parseDerivation(const Store & store, const string & s, std::string_view name) { Derivation drv; drv.name = name; @@ -197,7 +197,7 @@ static Derivation parseDerivation(const Store & store, const string & s, string } -Derivation readDerivation(const Store & store, const Path & drvPath, std::string name) +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name) { try { return parseDerivation(store, readFile(drvPath), name); @@ -465,7 +465,7 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { drv.name = name; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4694ec0dd..81c196ff7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -26,7 +26,7 @@ struct DerivationOutputFixed struct DerivationOutput { std::variant output; - StorePath path(const Store & store, string drvName) const; + StorePath path(const Store & store, std::string_view drvName) const; }; typedef std::map DerivationOutputs; @@ -45,7 +45,7 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; - string name; + std::string name; BasicDerivation() { } virtual ~BasicDerivation() { }; @@ -87,7 +87,7 @@ StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath, string name); +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); @@ -104,7 +104,7 @@ bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, string name); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); From 18152406ce9db9491385f2e67da6f7f78e8d98d5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:26:30 +0000 Subject: [PATCH 42/70] String .drv suffix to create derivation name --- src/libexpr/primops.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 11 ++++++++++- src/libstore/derivations.hh | 2 ++ src/nix-store/nix-store.cc | 2 +- src/nix/repl.cc | 2 +- 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 14a7deed5..21bd4fb52 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -113,7 +113,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args }; if (auto optStorePath = isValidDerivationInStore()) { auto storePath = *optStorePath; - Derivation drv = readDerivation(*state.store, realPath, std::string(storePath.name())); + Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath)); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4cf67064a..2ff53c964 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -474,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv, std::string(drvPath.name())); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 97fd21885..a85a70c0a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -218,7 +218,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), std::string(drvPath.name())); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -465,6 +465,15 @@ StringSet BasicDerivation::outputNames() const } +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { drv.name = name; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 81c196ff7..6d3b54d3f 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -64,6 +64,8 @@ struct BasicDerivation /* Return the output names of a derivation. */ StringSet outputNames() const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 39a47e19a..9605e4b32 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv, std::string(drvPath.name())); + readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath)); getBuildSettings(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bd72a3b19..74f0b3ec8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -483,7 +483,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath, std::string(state->store->parseStorePath(drvPath).name())); + auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath))); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); From a8d4707107c39d0258fef17b531ac031dd16ec88 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:54:12 +0000 Subject: [PATCH 43/70] Undo erroneous indentation change --- src/libstore/derivations.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a85a70c0a..f41edd81d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -396,7 +396,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } From 503b4256903b2cf414a6d697d19be40db299e2c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 15:56:20 +0000 Subject: [PATCH 44/70] DerivationOutputExtensional -> DerivationOutputInputAddressed Thanks @regnat for the great name. --- src/libexpr/primops.cc | 4 ++-- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 4 ++-- src/nix/develop.cc | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 21bd4fb52..5a9c17814 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -806,7 +806,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = StorePath::dummy, }, }); @@ -819,7 +819,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(outPath), }, }); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f41edd81d..d267468af 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -15,7 +15,7 @@ template overloaded(Ts...) -> overloaded; StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { return std::visit(overloaded { - [](DerivationOutputExtensional doi) { + [](DerivationOutputInputAddressed doi) { return doi.path; }, [&](DerivationOutputFixed dof) { @@ -142,7 +142,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream }; } else return DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(path), } }; @@ -450,7 +450,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) }; } else return DerivationOutput { - .output = DerivationOutputExtensional { + .output = DerivationOutputInputAddressed { .path = std::move(path), } }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d3b54d3f..fd8828373 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,7 +13,7 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutputExtensional +struct DerivationOutputInputAddressed { StorePath path; }; @@ -25,7 +25,7 @@ struct DerivationOutputFixed struct DerivationOutput { - std::variant output; + std::variant output; StorePath path(const Store & store, std::string_view drvName) const; }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e03d2e057..654f62f38 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputExtensional { + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = shellOutPath } }); drv.env["out"] = store->printStorePath(shellOutPath); From 886c91dfcc462d157dc2ce5265800e98d1bc45dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jul 2020 18:26:10 +0000 Subject: [PATCH 45/70] Try to fix perl bindings --- perl/lib/Nix/Store.xs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 6dba07549..522de5802 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath) HV * outputs = newHV(); for (auto & i : drv.outputs) - hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path(store(), drv.name)).c_str(), 0), 0); + hv_store( + outputs, i.first.c_str(), i.first.size(), + newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0), + 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); From 72f8771094d575e924846f16e5c60742eea9420b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 25 Jul 2020 18:02:42 -0500 Subject: [PATCH 46/70] Allow PRECOMPILE_HEADERS in cross-compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In cross, CXX will look like aarch64-unknown-linux-gnu-g++. We could run some command to check what kind of compiler it is, but for now we can just check if g++ is anywhere in the string. I couldn’t find any "ends with" for makefile, so it can be anywhere in CXX. --- mk/precompiled-headers.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1c0452dc2..500c99e4a 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(CXX), g++) + ifeq ($(findstring g++,$(CXX)), g++) GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch GLOBAL_ORDER_AFTER += $(GCH) - else ifeq ($(CXX), clang++) + else ifeq ($(findstring clang++,$(CXX)), clang++) GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch From f74243846512ffabf082985bca395890c97643e0 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:39:58 +0200 Subject: [PATCH 47/70] Merge legacy `fetchGit`-builtin with the generic `fetchTree`-function The original idea was to implement a git-fetcher in Nix's core that supports content hashes[1]. In #3549[2] it has been suggested to actually use `fetchTree` for this since it's a fairly generic wrapper over the new fetcher-API[3] and already supports content-hashes. This patch implements a new git-fetcher based on `fetchTree` by incorporating the following changes: * Removed the original `fetchGit`-implementation and replaced it with an alias on the `fetchTree` implementation. * Ensured that the `git`-fetcher from `libfetchers` always computes a content-hash and returns an "empty" revision on dirty trees (the latter one is needed to retain backwards-compatibility). * The hash-mismatch error in the fetcher-API exits with code 102 as it usually happens whenever a hash-mismatch is detected by Nix. * Removed the `flakes`-feature-flag: I didn't see a reason why this API is so tightly coupled to the flakes-API and at least `fetchGit` should remain usable without any feature-flags. * It's only possible to specify a `narHash` for a `git`-tree if either a `ref` or a `rev` is given[4]. * It's now possible to specify an URL without a protocol. If it's missing, `file://` is automatically added as it was the case in the original `fetchGit`-implementation. [1] https://github.com/NixOS/nix/pull/3216 [2] https://github.com/NixOS/nix/pull/3549#issuecomment-625194383 [3] https://github.com/NixOS/nix/pull/3459 [4] https://github.com/NixOS/nix/pull/3216#issuecomment-553956703 --- src/libexpr/primops/fetchGit.cc | 91 -------------------------------- src/libexpr/primops/fetchTree.cc | 62 +++++++++++++++++++--- src/libfetchers/fetchers.cc | 2 +- tests/fetchGit.sh | 12 +++-- tests/tarball.sh | 11 ++-- 5 files changed, 73 insertions(+), 105 deletions(-) delete mode 100644 src/libexpr/primops/fetchGit.cc diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc deleted file mode 100644 index 5013e74f0..000000000 --- a/src/libexpr/primops/fetchGit.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "hash.hh" -#include "fetchers.hh" -#include "url.hh" - -namespace nix { - -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional ref; - std::optional rev; - std::string name = "source"; - bool fetchSubmodules = false; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "submodules") - fetchSubmodules = state.forceBool(*attr.value, *attr.pos); - else - throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .errPos = *attr.pos - }); - } - - if (url.empty()) - throw EvalError({ - .hint = hintfmt("'url' argument required"), - .errPos = pos - }); - - } else - url = state.coerceToString(pos, *args[0], context, false, false); - - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); - - if (evalSettings.pureEval && !rev) - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - if (ref) attrs.insert_or_assign("ref", *ref); - if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit{true}); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); - - // FIXME: use name? - auto [tree, input2] = input.fetch(state.store); - - state.mkAttrs(v, 8); - auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - // Backward compatibility: set 'rev' to - // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); - // Backward compatibility: set 'revCount' to 0 for a dirty tree. - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - input2.getRevCount().value_or(0)); - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); - v.attrs->sort(); - - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); -} - -static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); - -} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5f480d919..887d2d14f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -36,6 +36,9 @@ void emitTreeAttrs( mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); } + if (input.getType() == "git") + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); + if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); @@ -48,10 +51,25 @@ void emitTreeAttrs( v.attrs->sort(); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +std::string fixURI(std::string uri, EvalState &state) { - settings.requireExperimentalFeature("flakes"); + state.checkURI(uri); + return uri.find("://") != std::string::npos ? uri : "file://" + uri; +} +void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +{ + string n(name); + attrs.emplace(name, n == "url" ? fixURI(v, state) : v); +} + +static void fetchTree( + EvalState &state, + const Pos &pos, + Value **args, + Value &v, + const std::optional type +) { fetchers::Input input; PathSet context; @@ -64,8 +82,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); + if (attr.value->type == tPath || attr.value->type == tString) + addURI( + state, + attrs, + attr.name, + state.coerceToString(*attr.pos, *attr.value, context, false, false) + ); + else if (attr.value->type == tString) + addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); else if (attr.value->type == tInt) @@ -75,6 +100,9 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V attr.name, showType(*attr.value)); } + if (type) + attrs.emplace("type", type.value()); + if (!attrs.count("type")) throw Error({ .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), @@ -82,8 +110,18 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V }); input = fetchers::Input::fromAttrs(std::move(attrs)); - } else - input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false)); + } else { + auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + + if (type == "git") { + fetchers::Attrs attrs; + attrs.emplace("type", "git"); + attrs.emplace("url", url); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + input = fetchers::Input::fromURL(url); + } + } if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; @@ -99,6 +137,12 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V emitTreeAttrs(state, tree, input2, v); } +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + fetchTree(state, pos, args, v, std::nullopt); +} + static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, @@ -178,7 +222,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args fetch(state, pos, args, v, "fetchTarball", true, "source"); } +static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +{ + fetchTree(state, pos, args, v, "git"); +} + static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); +static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit); } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c3ee9bf43..28db8aa9c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -134,7 +134,7 @@ std::pair Input::fetch(ref store) const if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true)); } diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 9faa5d9f6..1b9364e60 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD) # Fetch a worktree unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] @@ -87,8 +89,6 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] - # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] @@ -102,6 +102,12 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? +[[ "$status" = "102" ]] + +path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath") +[[ $path = $path5 ]] + # tarball-ttl should be ignored if we specify a rev echo delft > $repo/hello git -C $repo add hello @@ -123,7 +129,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --expr "(builtins.fetchGit $repo).rev or null") = null ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a diff --git a/tests/tarball.sh b/tests/tarball.sh index b3ec16d40..88a1a07a0 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -27,10 +27,13 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext From 189e6f5e1d949f50ab0b6e5acd25e230d206692d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jul 2020 20:51:16 +0200 Subject: [PATCH 48/70] Bump version to 3.0 Since there are some incompatible changes, it's better to bump the major version number. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 7208c2182..f398a2061 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.4 \ No newline at end of file +3.0 \ No newline at end of file From b8eea7e81af53905be7845dffc6d0a83ea8edc97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2020 13:27:56 +0200 Subject: [PATCH 49/70] Remove putBytes istream->tellg() returns -1 so we can't get the number of bytes written. Fixes 'uploaded 's3://nix-cache/nar/00819r9lp5kajr6baxfw5dhhc0cx8ndxaz43qmd2f0gn1hk1ynlp.nar.xz' (-1 bytes) in 11620 ms' messages. --- src/libstore/s3-binary-cache-store.cc | 7 ++----- src/libstore/s3-binary-cache-store.hh | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085..67935f3ba 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -343,13 +343,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - auto size = istream->tellg(); - - printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, size, duration); + printInfo("uploaded 's3://%s/%s' in %d ms", + bucketName, path, duration); stats.putTimeMs += duration; - stats.putBytes += size; stats.put++; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..b2b75d498 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -19,7 +19,6 @@ public: struct Stats { std::atomic put{0}; - std::atomic putBytes{0}; std::atomic putTimeMs{0}; std::atomic get{0}; std::atomic getBytes{0}; From e4940e90f399c5cde5f44f099ca5cdf5340d22b8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 28 Jul 2020 22:46:39 +0200 Subject: [PATCH 50/70] Restore backwards-compat for current `builtins.fetchGit` If a repo is dirty, it used to return a `rev` object with an "empty" sha1 (0000000000000000000000000000000000000000). Please note that this only applies for `builtins.fetchGit` and *not* for `builtins.fetchTree{ type = "git"; }`. --- src/libexpr/flake/flake.hh | 2 +- src/libexpr/primops/fetchTree.cc | 17 +++++++++++++---- tests/fetchGit.sh | 4 +++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 77f3abdeb..c2bb2888b 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -106,6 +106,6 @@ void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, const fetchers::Input & input, - Value & v); + Value & v, bool emptyRevFallback = false); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 887d2d14f..cddcf0e59 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -14,7 +14,8 @@ void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, const fetchers::Input & input, - Value & v) + Value & v, + bool emptyRevFallback) { assert(input.isImmutable()); @@ -34,6 +35,11 @@ void emitTreeAttrs( if (auto rev = input.getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); } if (input.getType() == "git") @@ -41,6 +47,8 @@ void emitTreeAttrs( if (auto revCount = input.getRevCount()) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); if (auto lastModified = input.getLastModified()) { mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); @@ -68,7 +76,8 @@ static void fetchTree( const Pos &pos, Value **args, Value &v, - const std::optional type + const std::optional type, + bool emptyRevFallback = false ) { fetchers::Input input; PathSet context; @@ -134,7 +143,7 @@ static void fetchTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v); + emitTreeAttrs(state, tree, input2, v, emptyRevFallback); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -224,7 +233,7 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) { - fetchTree(state, pos, args, v, "git"); + fetchTree(state, pos, args, v, "git", true); } static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 1b9364e60..cedd796f7 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -89,6 +89,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] @@ -129,7 +131,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --impure --expr "(builtins.fetchGit $repo).rev or null") = null ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a From c159f48a39835d5b2fe7c1ddd4467bc093ee251f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 00:24:55 +0200 Subject: [PATCH 51/70] Cleanup --- src/libstore/store-api.cc | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b6e6952b7..c804399d2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -729,9 +729,9 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); if (missing.empty()) return; @@ -748,29 +748,27 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st ThreadPool pool; - processGraph(pool, - PathSet(missing.begin(), missing.end()), + processGraph(pool, + StorePathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const StorePath & storePath) { + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); + auto info = srcStore->queryPathInfo(storePath); bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); - if (!dstStore->isValidPath(storePath)) { MaintainCount mc(nrRunning); showProgress(); @@ -780,7 +778,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } From 4c0077a07d2ee5362bd8ffb0ea9ee053c759303d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 00:48:39 +0200 Subject: [PATCH 52/70] Fix RemoteStore::addToStore() latency Since 6185d25e523a3cd223dd6f6aca10cf6ff15b4823, this was very latency-bound since it required a round-trip for every 32 KiB. So for example copying a 514 MiB closure over a virtual ethernet device with a articial delay of just 1 ms took 343s. Now it takes 2.7s. Fixes #3372. --- src/libstore/daemon.cc | 90 +++++++++++++++++++++++++++------ src/libstore/remote-store.cc | 81 +++++++++++++++++++++++++++-- src/libstore/worker-protocol.hh | 2 +- 3 files changed, 154 insertions(+), 19 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 503e04f92..478ae39ca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -86,7 +86,7 @@ struct TunnelLogger : public Logger } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -703,24 +703,84 @@ static void performOp(TunnelLogger * logger, ref store, if (!trusted) info.ultimate = false; - std::unique_ptr source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); - else { - StringSink saved; - TeeSource tee { from, saved }; - ParseSink ether; - parseDump(ether, tee); - source = std::make_unique(std::move(*saved.s)); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + + struct FramedSource : Source + { + Source & from; + bool eof = false; + std::vector pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } + }; + + logger->startWork(); + + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8d01c6667..4ed81d9d8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,9 +503,84 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + + std::exception_ptr ex; + + struct FramedSink : BufferedSink + { + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; + }; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + conn.processStderr(0, nullptr); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(conn, ex); + copyNAR(source, sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 8b538f6da..dcba73116 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x116 +#define PROTOCOL_VERSION 0x117 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From fa8515d7ec160938b0287f405a43aeeca971777e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 29 Jul 2020 00:57:20 -0500 Subject: [PATCH 53/70] Set regex_constants::match_continuous for quicker search match_continuous limits the search to the current start position, instead of searching the entire file. On libc++, this improves performance dramatically: $ time /nix/store/70ai68dfm6xbzwn26j5n4li9di52ylia-nix-3.0pre20200728_c159f48/bin/nix print-dev-env >/dev/null /nix/store/70ai68dfm6xbzwn26j5n4li9di52ylia-nix-3.0pre20200728_c159f48/bin/ni 2.39s user 0.19s system 64% cpu 4.032 total $ time /nix/store/cwjfxxlp83zln4mfyy1d2dbsx7f6s962-nix-3.0pre20200728_dirty/bin/nix print-dev-env >/dev/null /nix/store/cwjfxxlp83zln4mfyy1d2dbsx7f6s962-nix-3.0pre20200728_dirty/bin/nix 0.09s user 0.05s system 65% cpu 0.204 total Fixes #3874 --- src/nix/develop.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 2f590f53f..a0c119e43 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path) std::smatch match; - if (std::regex_search(pos, file.cend(), match, declareRegex)) { + if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { pos = match[0].second; exported.insert(match[1]); } - else if (std::regex_search(pos, file.cend(), match, varRegex)) { + else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { res.bashFunctions = std::string(pos, file.cend()); break; } From f63839bfa402f73e001a5ee412c3f6a6b95d1a38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 18:04:35 +0200 Subject: [PATCH 54/70] Cleanup --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4ed81d9d8..616081aa1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -545,7 +545,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, std::thread stderrThread([&]() { try { - conn.processStderr(0, nullptr); + conn.processStderr(); } catch (...) { ex = std::current_exception(); } From e4f6f8da77f8de0f19e81f1cbb60052ca92dd371 Mon Sep 17 00:00:00 2001 From: Mat Marini Date: Wed, 29 Jul 2020 20:50:06 -0400 Subject: [PATCH 55/70] Prevent nix-build from accepting --packages --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f77de56ea..d8f2fe6d7 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -192,7 +192,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (*arg == "--packages" || *arg == "-p") + else if (runEnv && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { From d9e23bfee2f1995b3047d67e578e2cfd4e94dea6 Mon Sep 17 00:00:00 2001 From: Mat Marini Date: Wed, 29 Jul 2020 20:53:02 -0400 Subject: [PATCH 56/70] Prevent nix-build from accepting --run/--command --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d8f2fe6d7..723db71fd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -174,7 +174,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--run-env") // obsolete runEnv = true; - else if (*arg == "--command" || *arg == "--run") { + else if (runEnv && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; From ebee2b7852a8c0441e9470a0df8ec3c71d322109 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 13:00:30 +0200 Subject: [PATCH 57/70] receiveContents(): unsigned int -> size_t --- src/libstore/nar-accessor.cc | 2 +- src/libutil/archive.cc | 2 +- src/libutil/archive.hh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index d884a131e..e438b5710 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -86,7 +86,7 @@ struct NarAccessor : public FSAccessor parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override + void receiveContents(unsigned char * data, size_t len) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51c88537e..c07e710cd 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -338,7 +338,7 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { writeFull(fd.get(), data, len); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 57780d16a..b6e1188c4 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -58,7 +58,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; + virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; @@ -77,7 +77,7 @@ struct RetrieveRegularNARSink : ParseSink regular = false; } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { sink(data, len); } From 3f6e88a5527dcc4d58e3147f78388a88eb8896e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 13:10:49 +0200 Subject: [PATCH 58/70] unsigned long long -> uint64_t --- src/build-remote/build-remote.cc | 8 ++++---- src/libexpr/primops.cc | 2 +- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 4 ++-- src/libstore/build.cc | 10 +++++----- src/libstore/daemon.cc | 2 +- src/libstore/gc.cc | 14 +++++++------- src/libstore/local-store.hh | 4 ++-- src/libstore/misc.cc | 6 +++--- src/libstore/nar-accessor.cc | 2 +- src/libstore/optimise-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 10 +++++----- src/libutil/archive.cc | 8 ++++---- src/libutil/archive.hh | 2 +- src/libutil/config.cc | 2 +- src/libutil/hash.hh | 4 ++-- src/libutil/json.cc | 2 +- src/libutil/serialise.hh | 16 ++++++++-------- src/libutil/util.cc | 8 ++++---- src/libutil/util.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 6 ++---- src/nix-store/nix-store.cc | 10 ++++------ src/nix/path-info.cc | 2 +- 26 files changed, 67 insertions(+), 71 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e07117496..3579d8fff 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -33,7 +33,7 @@ std::string escapeUri(std::string uri) static string currentLoad; -static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) +static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } @@ -119,7 +119,7 @@ static int _main(int argc, char * * argv) bool rightType = false; Machine * bestMachine = nullptr; - unsigned long long bestLoad = 0; + uint64_t bestLoad = 0; for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); @@ -130,8 +130,8 @@ static int _main(int argc, char * * argv) m.mandatoryMet(requiredFeatures)) { rightType = true; AutoCloseFD free; - unsigned long long load = 0; - for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + uint64_t load = 0; + for (uint64_t slot = 0; slot < m.maxJobs; ++slot) { auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5059286b3..a98cadc80 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -65,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(drvs); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52718c231..2b1f25ca3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() void printMissing(ref store, const std::vector & paths, Verbosity lvl) { - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); @@ -45,7 +45,7 @@ void printMissing(ref store, const std::vector & pa void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) + uint64_t downloadSize, uint64_t narSize, Verbosity lvl) { if (!willBuild.empty()) { if (willBuild.size() == 1) @@ -384,7 +384,7 @@ RunPager::~RunPager() } -string showBytes(unsigned long long bytes) +string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f558247c0..ffae5d796 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -47,7 +47,7 @@ void printMissing( void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); + uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end); @@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt; /* GC helpers. */ -string showBytes(unsigned long long bytes); +string showBytes(uint64_t bytes); struct GCResults; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d26f96b74..05235510d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -5038,7 +5038,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 478ae39ca..9ac7e7cca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -790,7 +790,7 @@ static void performOp(TunnelLogger * logger, ref store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); writeStorePaths(*store, to, willBuild); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index aaed5c218..e74382ed2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -500,7 +500,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 355c2814f..547fe589c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -29,8 +29,8 @@ struct Derivation; struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c4d22a634..a542a259d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -110,7 +110,7 @@ void Store::computeFSClosure(const StorePath & startPath, void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +122,8 @@ void Store::queryMissing(const std::vector & targets, { std::unordered_set done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index e438b5710..59ec164b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -79,7 +79,7 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); parents.top()->size = (uint64_t) size; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..e4b4b6213 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) +static string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 616081aa1..611f70783 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -782,7 +782,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c1b78b6a..eb6f225b1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -94,7 +94,7 @@ public: void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a7288d0cc..667dc4117 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -85,7 +85,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits::max()}; + uint64_t maxFreed{std::numeric_limits::max()}; }; @@ -97,7 +97,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; @@ -105,8 +105,8 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; @@ -610,7 +610,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index c07e710cd..ce7cf9754 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -150,17 +150,17 @@ static void skipGeneric(Source & source) static void parseContents(ParseSink & sink, Source & source, const Path & path) { - unsigned long long size = readLongLong(source); + uint64_t size = readLongLong(source); sink.preallocateContents(size); - unsigned long long left = size; + uint64_t left = size; std::vector buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); - if ((unsigned long long)n > left) n = left; + if ((uint64_t)n > left) n = left; source(buf.data(), n); sink.receiveContents(buf.data(), n); left -= n; @@ -323,7 +323,7 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(unsigned long long len) + void preallocateContents(uint64_t len) { #if HAVE_POSIX_FALLOCATE if (len) { diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index b6e1188c4..5665732d2 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -57,7 +57,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; + virtual void preallocateContents(uint64_t size) { }; virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8fc700a2b..8454037f2 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -284,7 +284,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; -template class BaseSetting; +template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 98ee1bed0..abcd58f24 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -111,7 +111,7 @@ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -typedef std::pair HashResult; +typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); @@ -141,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink private: HashType ht; Ctx * ctx; - unsigned long long bytes; + uint64_t bytes; public: HashSink(HashType ht); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 01331947e..104b99e6d 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -29,7 +29,7 @@ template<> void toJSON(std::ostream & str, const unsigned int & n) template<> void toJSON(std::ostream & str, const long & n) { str << n; } template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const uint64_t & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5d9acf887..c29c6b29b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -312,14 +312,14 @@ T readNum(Source & source) source(buf, sizeof(buf)); uint64_t n = - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); + ((uint64_t) buf[0]) | + ((uint64_t) buf[1] << 8) | + ((uint64_t) buf[2] << 16) | + ((uint64_t) buf[3] << 24) | + ((uint64_t) buf[4] << 32) | + ((uint64_t) buf[5] << 40) | + ((uint64_t) buf[6] << 48) | + ((uint64_t) buf[7] << 56); if (n > std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a0a8ff4d3..97d278581 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -374,7 +374,7 @@ void writeLine(int fd, string s) } -static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) { checkInterrupt(); @@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by } } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(const Path & path, uint64_t & bytesFreed) { Path dir = dirOf(path); if (dir == "") @@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) void deletePath(const Path & path) { - unsigned long long dummy; + uint64_t dummy; deletePath(path, dummy); } -void deletePath(const Path & path, unsigned long long & bytesFreed) +void deletePath(const Path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); bytesFreed = 0; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 42130f6dc..6850b5a7a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -125,7 +125,7 @@ void writeLine(int fd, string s); second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); -void deletePath(const Path & path, unsigned long long & bytesFreed); +void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 723db71fd..94412042f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -325,7 +325,7 @@ static void _main(int argc, char * * argv) auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index aa5ada3a6..bcf1d8518 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -67,10 +67,8 @@ static int _main(int argc, char * * argv) deleteOlderThan = getArg(*arg, arg, end); } else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--max-freed") { - long long maxFreed = getIntArg(*arg, arg, end, true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*arg == "--max-freed") + options.maxFreed = std::max(getIntArg(*arg, arg, end, true), (int64_t) 0); else return false; return true; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 76bf5154a..7b26970ef 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.push_back(store->followLinksToStorePathWithOutputs(i)); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*i == "--max-freed") + options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0); else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) if (!path.isDerivation()) paths2.push_back({path}); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 65f73cd94..0c12efaf0 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } - void printSize(unsigned long long value) + void printSize(uint64_t value) { if (!humanReadable) { std::cout << fmt("\t%11d", value); From a785b3eddf8c02750b1715939069d20980bd5125 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 15:27:28 +0200 Subject: [PATCH 59/70] Fix build --- src/libutil/config.cc | 2 +- src/libutil/json.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8454037f2..8fc700a2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -284,7 +284,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; -template class BaseSetting; +template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 104b99e6d..01331947e 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -29,7 +29,7 @@ template<> void toJSON(std::ostream & str, const unsigned int & n) template<> void toJSON(std::ostream & str, const long & n) { str << n; } template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const uint64_t & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } From 2f4250a4167f2b1c10e5fa9c6163f84bb9bbd740 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 11:33:22 -0500 Subject: [PATCH 60/70] Add "export" to Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a ‘nix export’ command which hooks into nix-bundle. It can be used in a similar way as nix-bundle, with the benefit of hooking into the new “app” functionality. For instance, $ nix export nixpkgs#jq $ ./jq --help jq - commandline JSON processor [version 1.6] ... $ scp jq machine-without-nix: $ ssh machine-without-nix ./jq jq - commandline JSON processor [version 1.6] ... Note that nix-bundle currently requires Linux to run. Other exporters might not have that requirement. “exporters” are meant to be reusable, so that, other repos can implement their own bundling. Fixes #3705 --- src/nix/export.cc | 126 ++++++++++++++++++++++++++++++++++++++++++++++ src/nix/flake.cc | 25 +++++++++ 2 files changed, 151 insertions(+) create mode 100644 src/nix/export.cc diff --git a/src/nix/export.cc b/src/nix/export.cc new file mode 100644 index 000000000..9e7816605 --- /dev/null +++ b/src/nix/export.cc @@ -0,0 +1,126 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "fs-accessor.hh" + +using namespace nix; + +struct CmdExport : InstallableCommand +{ + std::string exporter = "github:matthewbauer/nix-bundle"; + Path outLink; + + CmdExport() + { + addFlag({ + .longName = "exporter", + .description = "use custom exporter", + .labels = {"flake-url"}, + .handler = {&exporter}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + } + + std::string description() override + { + return "export an application out of the Nix store"; + } + + Examples examples() override + { + return { + Example{ + "To export Hello:", + "nix export hello" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + + auto app = installable->toApp(*evalState); + store->buildPaths(app.context); + + auto [exporterFlakeRef, exporterName] = parseFlakeRefWithFragment(exporter, absPath(".")); + const flake::LockFlags lockFlags{ .writeLockFile = false }; + auto exporter = InstallableFlake( + evalState, std::move(exporterFlakeRef), + Strings{exporterName == "" ? ("defaultExporter." + settings.thisSystem.get()) : exporterName}, + Strings({"exporters." + settings.thisSystem.get() + "."}), lockFlags); + + Value * arg = evalState->allocValue(); + evalState->mkAttrs(*arg, 1); + + PathSet context; + for (auto & i : app.context) + context.insert("=" + store->printStorePath(i.path)); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + + auto vRes = evalState->allocValue(); + evalState->callFunction(*exporter.toValue(*evalState).first, *arg, *vRes, noPos); + + if (!evalState->isDerivation(*vRes)) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); + if (i == vRes->attrs->end()) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + + i = vRes->attrs->find(evalState->sOutPath); + if (i == vRes->attrs->end()) + throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + + store->buildPaths({{drvPath}}); + + auto accessor = store->getFSAccessor(); + auto outPathS = store->printStorePath(outPath); + if (accessor->stat(outPathS).type != FSAccessor::tRegular) + throw Error("'%s' is not a file; an exporter must only create a single file", outPathS); + + auto info = store->queryPathInfo(outPath); + if (!info->references.empty()) + throw Error("'%s' has references; an exporter must not leave any references", outPathS); + + if (outLink == "") + outLink = baseNameOf(app.program); + + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(outLink), true); + } +}; + +static auto r2 = registerCommand("export"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 027a9871e..db3ebdc2a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -368,6 +368,21 @@ struct CmdFlakeCheck : FlakeCommand } }; + auto checkExporter = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda) + throw Error("exporter must be a function"); + if (!v.lambda.fun->formals || + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("args")) == v.lambda.fun->formals->argNames.end()) + throw Error("exporter must take formal arguments 'program' and 'args'"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -490,6 +505,16 @@ struct CmdFlakeCheck : FlakeCommand *attr.value, *attr.pos); } + else if (name == "defaultExporter") + checkExporter(name, vOutput, pos); + + else if (name == "exporters") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkExporter(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + else warn("unknown flake output '%s'", name); From 5d04a4db9b3d8cbfbaacbc48a587d21d90844871 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 11:45:47 -0500 Subject: [PATCH 61/70] Handle exporters checking correctly --- src/nix/flake.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index db3ebdc2a..cf7e4b7f5 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -374,9 +374,8 @@ struct CmdFlakeCheck : FlakeCommand if (v.type != tLambda) throw Error("exporter must be a function"); if (!v.lambda.fun->formals || - v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || - v.lambda.fun->formals->argNames.find(state->symbols.create("args")) == v.lambda.fun->formals->argNames.end()) - throw Error("exporter must take formal arguments 'program' and 'args'"); + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) + throw Error("exporter must take formal argument 'program'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -506,13 +505,21 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultExporter") - checkExporter(name, vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkExporter(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); + } else if (name == "exporters") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - checkExporter(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkExporter( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } } else From 52407f83a133ba605a1f0891c3812fa89038bba8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:03:57 -0500 Subject: [PATCH 62/70] exporter -> bundler --- src/nix/{export.cc => bundle.cc} | 42 ++++++++++++++++---------------- src/nix/flake.cc | 14 +++++------ 2 files changed, 28 insertions(+), 28 deletions(-) rename src/nix/{export.cc => bundle.cc} (67%) diff --git a/src/nix/export.cc b/src/nix/bundle.cc similarity index 67% rename from src/nix/export.cc rename to src/nix/bundle.cc index 9e7816605..e7338262b 100644 --- a/src/nix/export.cc +++ b/src/nix/bundle.cc @@ -6,18 +6,18 @@ using namespace nix; -struct CmdExport : InstallableCommand +struct CmdBundle : InstallableCommand { - std::string exporter = "github:matthewbauer/nix-bundle"; + std::string bundler = "github:matthewbauer/nix-bundle"; Path outLink; - CmdExport() + CmdBundle() { addFlag({ - .longName = "exporter", - .description = "use custom exporter", + .longName = "bundler", + .description = "use custom bundler", .labels = {"flake-url"}, - .handler = {&exporter}, + .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { completeFlakeRef(getStore(), prefix); }} @@ -35,15 +35,15 @@ struct CmdExport : InstallableCommand std::string description() override { - return "export an application out of the Nix store"; + return "bundle an application so that it works outside of the Nix store"; } Examples examples() override { return { Example{ - "To export Hello:", - "nix export hello" + "To bundle Hello:", + "nix bundle hello" }, }; } @@ -71,12 +71,12 @@ struct CmdExport : InstallableCommand auto app = installable->toApp(*evalState); store->buildPaths(app.context); - auto [exporterFlakeRef, exporterName] = parseFlakeRefWithFragment(exporter, absPath(".")); + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; - auto exporter = InstallableFlake( - evalState, std::move(exporterFlakeRef), - Strings{exporterName == "" ? ("defaultExporter." + settings.thisSystem.get()) : exporterName}, - Strings({"exporters." + settings.thisSystem.get() + "."}), lockFlags); + auto bundler = InstallableFlake( + evalState, std::move(bundlerFlakeRef), + Strings{bundlerName == "" ? ("defaultBundler." + settings.thisSystem.get()) : bundlerName}, + Strings({"bundlers." + settings.thisSystem.get() + "."}), lockFlags); Value * arg = evalState->allocValue(); evalState->mkAttrs(*arg, 1); @@ -87,21 +87,21 @@ struct CmdExport : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); auto vRes = evalState->allocValue(); - evalState->callFunction(*exporter.toValue(*evalState).first, *arg, *vRes, noPos); + evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); if (!evalState->isDerivation(*vRes)) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); if (i == vRes->attrs->end()) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a bderivation", bundler.what()); PathSet context2; StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); i = vRes->attrs->find(evalState->sOutPath); if (i == vRes->attrs->end()) - throw Error("the exporter '%s' does not produce a derivation", exporter.what()); + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); @@ -110,11 +110,11 @@ struct CmdExport : InstallableCommand auto accessor = store->getFSAccessor(); auto outPathS = store->printStorePath(outPath); if (accessor->stat(outPathS).type != FSAccessor::tRegular) - throw Error("'%s' is not a file; an exporter must only create a single file", outPathS); + throw Error("'%s' is not a file; a bundler must only create a single file", outPathS); auto info = store->queryPathInfo(outPath); if (!info->references.empty()) - throw Error("'%s' has references; an exporter must not leave any references", outPathS); + throw Error("'%s' has references; a bundler must not leave any references", outPathS); if (outLink == "") outLink = baseNameOf(app.program); @@ -123,4 +123,4 @@ struct CmdExport : InstallableCommand } }; -static auto r2 = registerCommand("export"); +static auto r2 = registerCommand("bundle"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cf7e4b7f5..1190d7997 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -368,14 +368,14 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkExporter = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); if (v.type != tLambda) - throw Error("exporter must be a function"); + throw Error("bundler must be a function"); if (!v.lambda.fun->formals || v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) - throw Error("exporter must take formal argument 'program'"); + throw Error("bundler must take formal argument 'program'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -504,19 +504,19 @@ struct CmdFlakeCheck : FlakeCommand *attr.value, *attr.pos); } - else if (name == "defaultExporter") + else if (name == "defaultBundler") for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); - checkExporter(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); + checkBundler(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); } - else if (name == "exporters") { + else if (name == "bundlers") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { checkSystemName(attr.name, *attr.pos); state->forceAttrs(*attr.value, *attr.pos); for (auto & attr2 : *attr.value->attrs) - checkExporter( + checkBundler( fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); } From 1a705637ce96c30424409d74cb3411554ae2847e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:05:51 -0500 Subject: [PATCH 63/70] Remove single file restriction for bundler --- src/nix/bundle.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e7338262b..677d42520 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -107,10 +107,7 @@ struct CmdBundle : InstallableCommand store->buildPaths({{drvPath}}); - auto accessor = store->getFSAccessor(); auto outPathS = store->printStorePath(outPath); - if (accessor->stat(outPathS).type != FSAccessor::tRegular) - throw Error("'%s' is not a file; a bundler must only create a single file", outPathS); auto info = store->queryPathInfo(outPath); if (!info->references.empty()) From 22fcfdf18af84bbbbc1bd416a9eed37a64e26eb5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:18:48 -0500 Subject: [PATCH 64/70] Address misc review --- src/nix/bundle.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 677d42520..95aed5269 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -9,7 +9,7 @@ using namespace nix; struct CmdBundle : InstallableCommand { std::string bundler = "github:matthewbauer/nix-bundle"; - Path outLink; + std::optional outLink; CmdBundle() { @@ -92,18 +92,18 @@ struct CmdBundle : InstallableCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - Bindings::iterator i = vRes->attrs->find(evalState->sDrvPath); - if (i == vRes->attrs->end()) - throw Error("the bundler '%s' does not produce a bderivation", bundler.what()); - - PathSet context2; - StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); - - i = vRes->attrs->find(evalState->sOutPath); - if (i == vRes->attrs->end()) + auto attr1 = vRes->attrs->find(evalState->sDrvPath); + if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - StorePath outPath = store->parseStorePath(evalState->coerceToPath(*i->pos, *i->value, context2)); + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + + auto attr2 = vRes->attrs->find(evalState->sOutPath); + if (!attr2) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); store->buildPaths({{drvPath}}); @@ -113,10 +113,10 @@ struct CmdBundle : InstallableCommand if (!info->references.empty()) throw Error("'%s' has references; a bundler must not leave any references", outPathS); - if (outLink == "") + if (!outLink) outLink = baseNameOf(app.program); - store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(outLink), true); + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink), true); } }; From fa2d1fb36e2ee92622379b9716f5b06e73aae72e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 15:37:05 -0500 Subject: [PATCH 65/70] Pass system to bundler --- src/nix/bundle.cc | 10 +++++++--- src/nix/flake.cc | 21 +++++++-------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 95aed5269..f2b78eea5 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -75,17 +75,21 @@ struct CmdBundle : InstallableCommand const flake::LockFlags lockFlags{ .writeLockFile = false }; auto bundler = InstallableFlake( evalState, std::move(bundlerFlakeRef), - Strings{bundlerName == "" ? ("defaultBundler." + settings.thisSystem.get()) : bundlerName}, - Strings({"bundlers." + settings.thisSystem.get() + "."}), lockFlags); + Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings({"bundlers."}), lockFlags); Value * arg = evalState->allocValue(); - evalState->mkAttrs(*arg, 1); + evalState->mkAttrs(*arg, 2); PathSet context; for (auto & i : app.context) context.insert("=" + store->printStorePath(i.path)); mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); + + arg->attrs->sort(); + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1190d7997..80d8654bc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -374,8 +374,9 @@ struct CmdFlakeCheck : FlakeCommand if (v.type != tLambda) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || - v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end()) - throw Error("bundler must take formal argument 'program'"); + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); throw; @@ -505,21 +506,13 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultBundler") - for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkBundler(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); - } + checkBundler(name, vOutput, pos); else if (name == "bundlers") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkBundler( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); - } + for (auto & attr : *vOutput.attrs) + checkBundler(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); } else From 0bdd6cf6ea9bc9779c1767e3cf0207b0373b3b52 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 16:37:44 -0500 Subject: [PATCH 66/70] Add test for builtins.path --- tests/filter-source.sh | 18 ++++++++++++------ tests/path.nix | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 tests/path.nix diff --git a/tests/filter-source.sh b/tests/filter-source.sh index 1f8dceee5..ba34d2eac 100644 --- a/tests/filter-source.sh +++ b/tests/filter-source.sh @@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak touch $TEST_ROOT/filterin/bla.c.bak ln -s xyzzy $TEST_ROOT/filterin/link -nix-build ./filter-source.nix -o $TEST_ROOT/filterout +checkFilter() { + test ! -e $1/foo/bar + test -e $1/xyzzy + test -e $1/bak + test ! -e $1/bla.c.bak + test ! -L $1/link +} -test ! -e $TEST_ROOT/filterout/foo/bar -test -e $TEST_ROOT/filterout/xyzzy -test -e $TEST_ROOT/filterout/bak -test ! -e $TEST_ROOT/filterout/bla.c.bak -test ! -L $TEST_ROOT/filterout/link +nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 +checkFilter $TEST_ROOT/filterout1 + +nix-build ./path.nix -o $TEST_ROOT/filterout2 +checkFilter $TEST_ROOT/filterout2 diff --git a/tests/path.nix b/tests/path.nix new file mode 100644 index 000000000..883c3c41b --- /dev/null +++ b/tests/path.nix @@ -0,0 +1,14 @@ +with import ./config.nix; + +mkDerivation { + name = "filter"; + builder = builtins.toFile "builder" "ln -s $input $out"; + input = + builtins.path { + path = ((builtins.getEnv "TEST_ROOT") + "/filterin"); + filter = path: type: + type != "symlink" + && baseNameOf path != "foo" + && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); + }; +} From cdc23866444b1ef863f917ec0adaaa9bec31126b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 30 Jul 2020 16:40:21 -0500 Subject: [PATCH 67/70] Make expectedHash optional in prim_path This fixes an error found in builtins.path that looks like: store path mismatch in (possibly filtered) path added from '/private/tmp/nix-shell.CyXViH/nix-test/filter-source/filterin' when no hash is specified --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a98cadc80..05d499d1f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1199,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash(htSHA256); + std::optional expectedHash; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); From 5a09eb86f17b7949b93724ecb51f1b909ea68592 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 29 Jun 2020 10:05:44 +0200 Subject: [PATCH 68/70] nix/build: add `--rebuild` option Occasionally, `nix-build --check` is fairly helpful and I'd like to be able to use this feature for flakes that need to be built with `nix build` as well. --- src/nix/build.cc | 9 ++++++++- src/nix/command.hh | 3 ++- src/nix/installables.cc | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 0f7e0e123..bec25d8d8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,6 +9,7 @@ using namespace nix; struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; + BuildMode buildMode = bmNormal; CmdBuild() { @@ -26,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "do not create a symlink to the build result", .handler = {&outLink, Path("")}, }); + + addFlag({ + .longName = "rebuild", + .description = "Rebuild an already built package locally and compare the result to the existing store-paths.", + .handler = {&buildMode, bmCheck}, + }); } std::string description() override @@ -53,7 +60,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile void run(ref store) override { - auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables); + auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); if (dryRun) return; diff --git a/src/nix/command.hh b/src/nix/command.hh index 856721ebf..bc46a2028 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -5,6 +5,7 @@ #include "common-eval-args.hh" #include "path.hh" #include "flake/lockfile.hh" +#include "store-api.hh" #include @@ -185,7 +186,7 @@ static RegisterCommand registerCommand(const std::string & name) } Buildables build(ref store, Realise mode, - std::vector> installables); + std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, Realise mode, OperateOn operateOn, diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a13e5a3df..56c62dbfe 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -642,7 +642,7 @@ std::shared_ptr SourceExprCommand::parseInstallable( } Buildables build(ref store, Realise mode, - std::vector> installables) + std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; @@ -668,7 +668,7 @@ Buildables build(ref store, Realise mode, if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild); + store->buildPaths(pathsToBuild, bMode); return buildables; } From f56dfce7342d66fe81796f039afc283f84ca10fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2020 17:30:12 +0200 Subject: [PATCH 69/70] nix bundle: Set category --- src/nix/bundle.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index f2b78eea5..eb3339f5d 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -48,6 +48,8 @@ struct CmdBundle : InstallableCommand }; } + Category category() override { return catSecondary; } + Strings getDefaultFlakeAttrPaths() override { Strings res{"defaultApp." + settings.thisSystem.get()}; From a3f9625818ecb0c8a3c22c191340dac5a3120bb5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2020 17:32:16 +0200 Subject: [PATCH 70/70] Tweak description --- src/nix/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index bec25d8d8..613cc15eb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -30,7 +30,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile addFlag({ .longName = "rebuild", - .description = "Rebuild an already built package locally and compare the result to the existing store-paths.", + .description = "rebuild an already built package and compare the result to the existing store paths", .handler = {&buildMode, bmCheck}, }); }