diff --git a/.dir-locals.el b/.dir-locals.el index 8b21d4e40..a2d1dc48d 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -1,6 +1,7 @@ ((c++-mode . ( (c-file-style . "k&r") (c-basic-offset . 4) + (c-block-comment-prefix . " ") (indent-tabs-mode . nil) (tab-width . 4) (show-trailing-whitespace . t) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7755466a0..47fa041e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,3 +12,13 @@ jobs: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v10 - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings + macos_perf_test: + runs-on: macos-latest + steps: + - name: Disable syspolicy assessments + run: | + spctl --status + sudo spctl --master-disable + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v10 + - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..6cb69d0a9 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional ca) { return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..22a039242 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 58cf08763..4d5eec7bf 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -365,6 +365,9 @@ public: Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7ecaa8743..d1aca9790 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -969,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1040,8 +1040,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1091,16 +1090,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref(readFile(srcPath)); + /* For computing the NAR hash. */ + auto sha256Sink = std::make_unique(htSHA256); - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); + /* For computing the store path. In recursive SHA-256 mode, this + is the same as the NAR hash, so no need to do it again. */ + std::unique_ptr hashSink = + method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? nullptr + : std::make_unique(hashAlgo); + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = true; + std::string nar; + + auto source = sinkToSource([&](Sink & sink) { + + LambdaSink sink2([&](const unsigned char * buf, size_t len) { + (*sha256Sink)(buf, len); + if (hashSink) (*hashSink)(buf, len); + + if (inMemory) { + if (nar.size() + len > settings.narBufferSize) { + inMemory = false; + sink << 1; + sink((const unsigned char *) nar.data(), nar.size()); + nar.clear(); + } else { + nar.append((const char *) buf, len); + } + } + + if (!inMemory) sink(buf, len); + }); + + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink2, filter); + else + readFile(srcPath, sink2); + }); + + std::unique_ptr delTempDir; + Path tempPath; + + try { + /* Wait for the source coroutine to give us some dummy + data. This is so that we don't create the temporary + directory if the NAR fits in memory. */ + readInt(*source); + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique(tempDir); + tempPath = tempDir + "/x"; + + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, *source); + else + writeFile(tempPath, *source); + + } catch (EndOfFile &) { + if (!inMemory) throw; + /* The NAR fits in memory, so we didn't do restorePath(). */ + } + + auto sha256 = sha256Sink->finish(); + + Hash hash = hashSink ? hashSink->finish().first : sha256.first; + + auto dstPath = makeFixedOutputPath(method, hash, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + auto realPath = Store::toRealPath(dstPath); + + PathLocks outputLock({realPath}); + + if (repair || !isValidPath(dstPath)) { + + deletePath(realPath); + + autoGC(); + + if (inMemory) { + /* Restore from the NAR in memory. */ + StringSource source(nar); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, source); + else + writeFile(realPath, source); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } + + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath + + optimisePath(realPath); + + ValidPathInfo info(dstPath); + info.narHash = sha256.first; + info.narSize = sha256.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; } @@ -1114,8 +1222,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..8d46bb436 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "archive.hh" #include @@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional expectedCAHash) +{ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' three + times. */ + + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + + auto hash = method == FileIngestionMethod::Recursive + ? hashAlgo == htSHA256 + ? narHash + : hashPath(hashAlgo, srcPath).first + : hashFile(hashAlgo, srcPath); + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + ValidPathInfo info(makeFixedOutputPath(method, hash, name)); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) : Config(params) , state({(size_t) pathInfoCacheSize}) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b122e05d6..b1dd1f478 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -450,6 +450,13 @@ public: FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional expectedCAHash = {}); + // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..961e7fb6d 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,14 +153,15 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + std::optional expectedHash; + Hash hash; std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); if (store->isValidPath(*storePath)) - hash = expectedHash; + hash = *expectedHash; else storePath.reset(); } @@ -200,22 +201,12 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient; addToStore() will also hash - this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error("hash mismatch for '%1%'", uri); - - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); - - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 2a2f16ac1..39a47e19a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); }