diff --git a/.dir-locals.el b/.dir-locals.el index 8b21d4e40..a2d1dc48d 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -1,6 +1,7 @@ ((c++-mode . ( (c-file-style . "k&r") (c-basic-offset . 4) + (c-block-comment-prefix . " ") (indent-tabs-mode . nil) (tab-width . 4) (show-trailing-whitespace . t) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index baefe0f12..91ae896d6 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -170,15 +170,5 @@ $channelsBucket->add_key( chdir("/home/eelco/Dev/nix-pristine") or die; system("git remote update origin") == 0 or die; system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die; - -# Update the website. -my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine"; - -system("cd $siteDir && git pull") == 0 or die; - -write_file("$siteDir/nix-release.tt", - "[%-\n" . - "latestNixVersion = \"$version\"\n" . - "-%]\n"); - -system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die; +system("git push --tags") == 0 or die; +system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7..f14c3f73f 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs); + store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5510c64e9..7a2f55504 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -381,7 +381,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) if (store->isInStore(r.second)) { StorePathSet closure; - store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); + store->computeFSClosure(store->toStorePath(r.second).first, closure); for (auto & path : closure) allowedPaths->insert(store->printStorePath(path)); } else diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 539a13f30..9f877f765 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -871,10 +871,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V .hint = hintfmt("path '%1%' is not in the Nix store", path), .errPos = pos }); - Path path2 = state.store->toStorePath(path); + auto path2 = state.store->toStorePath(path).first; if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(path2)); - context.insert(path2); + state.store->ensurePath(path2); + context.insert(state.store->printStorePath(path2)); mkString(v, path, context); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..b791c125b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -57,6 +58,13 @@ void BinaryCacheStore::init() } } +void BinaryCacheStore::upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType) +{ + upsertFile(path, std::make_shared(std::move(data)), mimeType); +} + void BinaryCacheStore::getFile(const std::string & path, Callback> callback) noexcept { @@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) +AutoCloseFD openFile(const Path & path) { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref(narSource.drain()); + auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return fd; +} - if (!repair && isValidPath(info.path)) return; +struct FileSource : FdSource +{ + AutoCloseFD fd2; + + FileSource(const Path & path) + : fd2(openFile(path)) + { + fd = fd2.get(); + } +}; + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + assert(info.narHash && info.narSize); + + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + auto [fdTemp, fnTemp] = createTempFile(); + + auto now1 = std::chrono::steady_clock::now(); + + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ + HashSink fileHashSink(htSHA256); + std::shared_ptr narAccessor; + { + FdSink fileSink(fdTemp.get()); + TeeSink teeSink(fileSink, fileHashSink); + auto compressionSink = makeCompressionSink(compression, teeSink); + TeeSource teeSource(narSource, *compressionSink); + narAccessor = makeNarAccessor(teeSource); + compressionSink->finish(); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto narInfo = make_ref(info); + narInfo->narSize = info.narSize; + narInfo->narHash = info.narHash; + narInfo->compression = compression; + auto [fileHash, fileSize] = fileHashSink.finish(); + narInfo->fileHash = fileHash; + narInfo->fileSize = fileSize; + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); + + auto duration = std::chrono::duration_cast(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), info.narSize, + ((1.0 - (double) fileSize / info.narSize) * 100.0), + duration); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - assert(nar->compare(0, narMagic.size(), narMagic) == 0); - - auto narInfo = make_ref(info); - - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); - - if (info.narHash && info.narHash != narInfo->narHash) - throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); - - auto accessor_ = std::dynamic_pointer_cast(accessor); - - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref(narAccessor), "", true); } } upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", - printStorePath(narInfo->path), narInfo->narSize, - ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), - duration); - - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ @@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + upsertFile(narInfo->url, + std::make_shared(fnTemp, std::ios_base::in), + "application/x-nix-nar"); } else stats.narWriteAverted++; - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteBytes += info.narSize; + stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -351,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath ValidPathInfo info(makeFixedOutputPath(method, h, name)); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); return std::move(info.path); } @@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s StringSink sink; dumpString(s, sink); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); } return std::move(info.path); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7a..9bcdf5901 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -36,9 +36,13 @@ public: virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr> istream, const std::string & mimeType) = 0; + void upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType); + /* Note: subclasses must implement at least one of the two following getFile() methods. */ @@ -75,8 +79,7 @@ public: { unsupported("queryPathFromHashPart"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0ef2f288f..ac2e67574 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2044,7 +2044,7 @@ void DerivationGoal::startBuilder() auto storePathS = *i++; if (!worker.store.isInStore(storePathS)) throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.parseStorePath(worker.store.toStorePath(storePathS)); + auto storePath = worker.store.toStorePath(storePathS).first; /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, @@ -2083,7 +2083,7 @@ void DerivationGoal::startBuilder() for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { throw Error("while processing 'sandbox-paths': %s", e.what()); @@ -2768,10 +2768,9 @@ struct RestrictedStore : public LocalFSStore { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0) override + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override { - next->addToStore(info, narSource, repair, checkSigs, accessor); + next->addToStore(info, narSource, repair, checkSigs); goal.addDependency(info.path); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..6cb69d0a9 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional ca) { return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..22a039242 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ebc4d0285..db7139374 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -391,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref store, } HashType hashAlgo = parseHashType(s); - TeeSource savedNAR(from); + StringSink savedNAR; + TeeSource savedNARSource(from, savedNAR); RetrieveRegularNARSink savedRegular; if (method == FileIngestionMethod::Recursive) { @@ -399,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref store, a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); + parseDump(sink, savedNARSource); } else parseDump(savedRegular, from); @@ -407,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref store, if (!savedRegular.regular) throw Error("regular file expected"); auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, baseName, method, hashAlgo); @@ -442,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - auto paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; @@ -731,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeSink tee(from); + TeeParseSink tee(from); parseDump(tee, tee.source); - saved = std::move(*tee.source.data); + saved = std::move(*tee.saved.s); source = std::make_unique(saved); } @@ -741,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref store, // FIXME: race if addToStore doesn't read source? store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + dontCheckSigs ? NoCheckSigs : CheckSigs); logger->stopWork(); break; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..f325e511a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,7 +4,6 @@ #include "util.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" -#include "istringstream_nocopy.hh" namespace nix { @@ -101,7 +100,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) { expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); auto hashAlgo = parseString(str); @@ -129,10 +128,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream } -static Derivation parseDerivation(const Store & store, const string & s) +static Derivation parseDerivation(const Store & store, std::string && s) { Derivation drv; - istringstream_nocopy str(s); + std::istringstream str(std::move(s)); expect(str, "Derive(["); /* Parse the list of outputs. */ diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..082d0f1d1 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -7,24 +7,6 @@ namespace nix { -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - void Store::exportPaths(const StorePathSet & paths, Sink & sink) { auto sorted = topoSortPaths(paths); @@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashAndWriteSink hashAndWriteSink(sink); + HashSink hashSink(htSHA256); + TeeSink teeSink(sink, hashSink); - narFromPath(path, hashAndWriteSink); + narFromPath(path, teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); + Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); - hashAndWriteSink + teeSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); - hashAndWriteSink + writeStorePaths(*this, teeSink, info->references); + teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; } -StorePaths Store::importPaths(Source & source, std::shared_ptr accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) { StorePaths res; while (true) { @@ -77,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); + TeeParseSink tee(source); parseDump(tee, tee.source); uint32_t magic = readInt(source); @@ -94,16 +77,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.saved.s); + info.narSize = tee.saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; - addToStore(info, source, NoRepair, checkSigs, accessor); + auto source = StringSource { *tee.saved.s }; + addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 57fb20845..aaed5c218 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { - auto storePath = maybeParseStorePath(toStorePath(target)); - if (storePath && isValidPath(*storePath)) - roots[std::move(*storePath)].emplace(path); - else - printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + try { + auto storePath = toStorePath(target).first; + if (isValidPath(storePath)) + roots[std::move(storePath)].emplace(path); + else + printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + } catch (BadStorePath &) { } }; try { @@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (auto & [target, links] : unchecked) { if (!isInStore(target)) continue; - Path pathS = toStorePath(target); - if (!isStorePath(pathS)) continue; - auto path = parseStorePath(pathS); - if (!isValidPath(path)) continue; - debug("got additional root '%1%'", pathS); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); + try { + auto path = toStorePath(target).first; + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { } } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 101633869..d47e0b6b5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -366,6 +366,9 @@ public: Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; + Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 451a64785..c1ceb08cf 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -100,11 +100,11 @@ protected: } void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr> istream, const std::string & mimeType) override { auto req = FileTransferRequest(cacheUri + "/" + path); - req.data = std::make_shared(data); // FIXME: inefficient + req.data = std::make_shared(StreamToSourceAdapter(istream).drain()); req.mimeType = mimeType; try { getFileTransfer()->upload(req); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..a8bd8a972 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -126,8 +126,7 @@ struct LegacySSHStore : public Store } void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) override + RepairFlag repair, CheckSigsFlag checkSigs) override { debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 48aca478c..87d8334d7 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -31,8 +31,18 @@ protected: bool fileExists(const std::string & path) override; void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override; + std::shared_ptr> istream, + const std::string & mimeType) override + { + auto path2 = binaryCacheDir + "/" + path; + Path tmp = path2 + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + StreamToSourceAdapter source(istream); + writeFile(tmp, source); + if (rename(tmp.c_str(), path2.c_str())) + throw SysError("renaming '%1%' to '%2%'", tmp, path2); + del.cancel(); + } void getFile(const std::string & path, Sink & sink) override { @@ -52,7 +62,9 @@ protected: if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); + paths.insert(parseStorePath( + storeDir + "/" + entry.name.substr(0, entry.name.size() - 8) + + "-" + MissingName)); } return paths; @@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init() BinaryCacheStore::init(); } -static void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path); - del.cancel(); -} - bool LocalBinaryCacheStore::fileExists(const std::string & path) { return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) -{ - atomicWrite(binaryCacheDir + "/" + path, data); -} - static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index dd96d2578..2f1d9663a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { - Path storePath = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + auto storePath = store->toStorePath(path).first; + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 12a6a03c3..1db08b275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -962,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { if (!info.narHash) throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); @@ -976,7 +976,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1047,8 +1047,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1098,16 +1097,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref(readFile(srcPath)); + /* For computing the NAR hash. */ + auto sha256Sink = std::make_unique(htSHA256); - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); + /* For computing the store path. In recursive SHA-256 mode, this + is the same as the NAR hash, so no need to do it again. */ + std::unique_ptr hashSink = + method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? nullptr + : std::make_unique(hashAlgo); + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = true; + std::string nar; + + auto source = sinkToSource([&](Sink & sink) { + + LambdaSink sink2([&](const unsigned char * buf, size_t len) { + (*sha256Sink)(buf, len); + if (hashSink) (*hashSink)(buf, len); + + if (inMemory) { + if (nar.size() + len > settings.narBufferSize) { + inMemory = false; + sink << 1; + sink((const unsigned char *) nar.data(), nar.size()); + nar.clear(); + } else { + nar.append((const char *) buf, len); + } + } + + if (!inMemory) sink(buf, len); + }); + + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink2, filter); + else + readFile(srcPath, sink2); + }); + + std::unique_ptr delTempDir; + Path tempPath; + + try { + /* Wait for the source coroutine to give us some dummy + data. This is so that we don't create the temporary + directory if the NAR fits in memory. */ + readInt(*source); + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique(tempDir); + tempPath = tempDir + "/x"; + + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, *source); + else + writeFile(tempPath, *source); + + } catch (EndOfFile &) { + if (!inMemory) throw; + /* The NAR fits in memory, so we didn't do restorePath(). */ + } + + auto sha256 = sha256Sink->finish(); + + Hash hash = hashSink ? hashSink->finish().first : sha256.first; + + auto dstPath = makeFixedOutputPath(method, hash, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + auto realPath = Store::toRealPath(dstPath); + + PathLocks outputLock({realPath}); + + if (repair || !isValidPath(dstPath)) { + + deletePath(realPath); + + autoGC(); + + if (inMemory) { + /* Restore from the NAR in memory. */ + StringSource source(nar); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, source); + else + writeFile(realPath, source); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } + + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath + + optimisePath(realPath); + + ValidPathInfo info(dstPath); + info.narHash = sha256.first; + info.narSize = sha256.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; } @@ -1121,8 +1229,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ff36cb00e..c0e5d0286 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -143,8 +143,7 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ca663d837..d884a131e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -18,7 +18,7 @@ struct NarMember /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start = 0, size = 0; + uint64_t start = 0, size = 0; std::string target; @@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, StringSource + struct NarIndexer : ParseSink, Source { NarAccessor & acc; + Source & source; std::stack parents; - std::string currentStart; bool isExec = false; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) + uint64_t pos = 0; + + NarIndexer(NarAccessor & acc, Source & source) + : acc(acc), source(source) { } void createMember(const Path & path, NarMember member) { @@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor void preallocateContents(unsigned long long size) override { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits::max()); - parents.top()->size = (size_t)size; + assert(size <= std::numeric_limits::max()); + parents.top()->size = (uint64_t) size; parents.top()->start = pos; } void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } + { } void createSymlink(const Path & path, const string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + + size_t read(unsigned char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } }; NarAccessor(ref nar) : nar(nar) { - NarIndexer indexer(*this, *nar); + StringSource source(*nar); + NarIndexer indexer(*this, source); + parseDump(indexer, indexer); + } + + NarAccessor(Source & source) + { + NarIndexer indexer(*this, source); parseDump(indexer, indexer); } @@ -219,6 +228,11 @@ ref makeNarAccessor(ref nar) return make_ref(nar); } +ref makeNarAccessor(Source & source) +{ + return make_ref(source); +} + ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 2871199de..8af1272f6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -6,10 +6,14 @@ namespace nix { +struct Source; + /* Return an object that provides access to the contents of a NAR file. */ ref makeNarAccessor(ref nar); +ref makeNarAccessor(Source & source); + /* Create a NAR accessor from a NAR listing (in the format produced by listNar()). The callback getNarBytes(offset, length) is used by the readFile() method of the accessor to get the contents of files diff --git a/src/libstore/path.cc b/src/libstore/path.cc index b3d8ce95c..dc9dc3897 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,8 +2,6 @@ namespace nix { -MakeError(BadStorePath, Error); - static void checkName(std::string_view path, std::string_view name) { if (name.empty()) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index bd698d781..2d02a181b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) createDirs(cacheDir); } -Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); + return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, +void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, ref narAccessor) { - nars.emplace(storePath, narAccessor); + nars.emplace(hashPart, narAccessor); if (cacheDir != "") { try { std::ostringstream str; JSONPlaceholder jsonRoot(str); listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(storePath, "ls"), str.str()); + writeFile(makeCacheFile(hashPart, "ls"), str.str()); /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(storePath, "nar"), nar); + writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { ignoreException(); @@ -47,23 +47,22 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) { auto path = canonPath(path_); - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); + auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - auto i = nars.find(storePath); + auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) return {i->second, restPath}; StringSink sink; std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { - listing = nix::readFile(makeCacheFile(storePath, "ls")); + listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { @@ -81,7 +80,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) return buf; }); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } @@ -90,15 +89,15 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) *sink.s = nix::readFile(cacheFile); auto narAccessor = makeNarAccessor(sink.s); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } } - store->narFromPath(store->parseStorePath(storePath), sink); + store->narFromPath(storePath, sink); auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath, *sink.s, narAccessor); + addToCache(storePath.hashPart(), *sink.s, narAccessor); return {narAccessor, restPath}; } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 4afb3be95..347cf5764 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor { ref store; - std::map> nars; + std::map> nars; Path cacheDir; @@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor friend class BinaryCacheStore; - Path makeCacheFile(const Path & storePath, const std::string & ext); + Path makeCacheFile(std::string_view hashPart, const std::string & ext); - void addToCache(const Path & storePath, const std::string & nar, + void addToCache(std::string_view hashPart, const std::string & nar, ref narAccessor); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 118aadf7e..9af4364b7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -466,7 +466,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fb2052752..3c1b78b6a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -60,8 +60,7 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 427dd48ce..1b7dff085 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,7 +7,6 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" -#include "istringstream_nocopy.hh" #include #include @@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr transferManager; std::once_flag transferManagerCreated; - void uploadFile(const std::string & path, const std::string & data, + void uploadFile(const std::string & path, + std::shared_ptr> istream, const std::string & mimeType, const std::string & contentEncoding) { - auto stream = std::make_shared(data); - auto maxThreads = std::thread::hardware_concurrency(); static std::shared_ptr @@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr transferHandle = transferManager->UploadFile( - stream, bucketName, path, mimeType, + istream, bucketName, path, mimeType, Aws::Map(), nullptr /*, contentEncoding */); @@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (contentEncoding != "") request.SetContentEncoding(contentEncoding); - auto stream = std::make_shared(data); - - request.SetBody(stream); + request.SetBody(istream); auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); @@ -347,25 +343,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % - bucketName % path % data.size() % duration); + auto size = istream->tellg(); + + printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, size, duration); stats.putTimeMs += duration; - stats.putBytes += data.size(); + stats.putBytes += size; stats.put++; } - void upsertFile(const std::string & path, const std::string & data, + void upsertFile(const std::string & path, + std::shared_ptr> istream, const std::string & mimeType) override { + auto compress = [&](std::string compression) + { + auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); + return std::make_shared(std::move(*compressed)); + }; + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + uploadFile(path, compress(lsCompression), mimeType, lsCompression); else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + uploadFile(path, compress(logCompression), mimeType, logCompression); else - uploadFile(path, data, mimeType, ""); + uploadFile(path, istream, mimeType, ""); } void getFile(const std::string & path, Sink & sink) override @@ -410,7 +415,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); } marker = res.GetNextMarker(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..5b9f79049 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "archive.hh" #include @@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const } -Path Store::toStorePath(const Path & path) const +std::pair Store::toStorePath(const Path & path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) - return path; + return {parseStorePath(path), ""}; else - return Path(path, 0, slash); + return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; } @@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw NotInStore("path '%1%' is not in the Nix store", path); + throw BadStorePath("path '%1%' is not in the Nix store", path); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return parseStorePath(toStorePath(followLinksToStore(path))); + return toStorePath(followLinksToStore(path)).first; } @@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional expectedCAHash) +{ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' three + times. */ + + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + + auto hash = method == FileIngestionMethod::Recursive + ? hashAlgo == htSHA256 + ? narHash + : hashPath(hashAlgo, srcPath).first + : hashFile(hashAlgo, srcPath); + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + ValidPathInfo info(makeFixedOutputPath(method, hash, name)); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) : Config(params) , state({(size_t) pathInfoCacheSize}) @@ -316,6 +351,14 @@ ref Store::queryPathInfo(const StorePath & storePath) } +static bool goodStorePath(const StorePath & expected, const StorePath & actual) +{ + return + expected.hashPart() == actual.hashPart() + && (expected.name() == Store::MissingName || expected.name() == actual.name()); +} + + void Store::queryPathInfo(const StorePath & storePath, Callback> callback) noexcept { @@ -343,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - res.second->path != storePath) + !goodStorePath(storePath, res.second->path)) throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref(res.second)); @@ -355,7 +398,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -368,9 +411,11 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - if (!info || info->path != parseStorePath(storePath)) { + auto storePath = parseStorePath(storePathS); + + if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); + throw InvalidPath("path '%s' is not valid", storePathS); } (*callbackPtr)(ref(info)); @@ -515,7 +560,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b122e05d6..a4be0411e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -31,7 +31,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(NotInStore, Error); +MakeError(BadStorePath, Error); class FSAccessor; @@ -317,9 +317,9 @@ public: the Nix store. */ bool isStorePath(std::string_view path) const; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + /* Split a path like /nix/store/-/ into + /nix/store/- and /. */ + std::pair toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -384,13 +384,16 @@ public: SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/ rather than + backends, the name part of store paths may be replaced by 'x' + (i.e. you'll get /nix/store/-x rather than /nix/store/-). Use queryPathInfo() to obtain the - full store path. */ + full store path. FIXME: should return a set of + std::variant to get rid of this hack. */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + constexpr static const char * MissingName = "x"; + /* Query information about a valid path. It is permitted to omit the name part of the store path. */ ref queryPathInfo(const StorePath & path); @@ -439,8 +442,7 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0) = 0; + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. @@ -450,6 +452,13 @@ public: FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional expectedCAHash = {}); + // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) @@ -616,8 +625,7 @@ public: the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - StorePaths importPaths(Source & source, std::shared_ptr accessor, - CheckSigsFlag checkSigs = CheckSigs); + StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats { diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index a38c2d798..ae741f867 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -11,7 +11,7 @@ namespace nix { #define ANSI_GREEN "\e[32;1m" #define ANSI_YELLOW "\e[33;1m" #define ANSI_BLUE "\e[34;1m" -#define ANSI_MAGENTA "\e[35m;1m" -#define ANSI_CYAN "\e[36m;1m" +#define ANSI_MAGENTA "\e[35;1m" +#define ANSI_CYAN "\e[36;1m" } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe2536..302b1bb18 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,11 +63,12 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeSink : ParseSink +struct TeeParseSink : ParseSink { + StringSink saved; TeeSource source; - TeeSink(Source & source) : source(source) { } + TeeParseSink(Source & source) : source(source, saved) { } }; void parseDump(ParseSink & sink, Source & source); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index f954b2923..5578a618e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -8,7 +8,6 @@ #include "hash.hh" #include "archive.hh" #include "util.hh" -#include "istringstream_nocopy.hh" #include #include diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh deleted file mode 100644 index f7beac578..000000000 --- a/src/libutil/istringstream_nocopy.hh +++ /dev/null @@ -1,92 +0,0 @@ -/* This file provides a variant of std::istringstream that doesn't - copy its string argument. This is useful for large strings. The - caller must ensure that the string object is not destroyed while - it's referenced by this object. */ - -#pragma once - -#include -#include - -template , class Allocator = std::allocator> -class basic_istringbuf_nocopy : public std::basic_streambuf -{ -public: - typedef std::basic_string string_type; - - typedef typename std::basic_streambuf::off_type off_type; - - typedef typename std::basic_streambuf::pos_type pos_type; - - typedef typename std::basic_streambuf::int_type int_type; - - typedef typename std::basic_streambuf::traits_type traits_type; - -private: - const string_type & s; - - off_type off; - -public: - basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} - { - } - -private: - pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) - { - if (which & std::ios_base::in) { - this->off = dir == std::ios_base::beg - ? off - : (dir == std::ios_base::end - ? s.size() + off - : this->off + off); - } - return pos_type(this->off); - } - - pos_type seekpos(pos_type pos, std::ios_base::openmode which) - { - return seekoff(pos, std::ios_base::beg, which); - } - - std::streamsize showmanyc() - { - return s.size() - off; - } - - int_type underflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off]); - } - - int_type uflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off++]); - } - - int_type pbackfail(int_type ch) - { - if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) - return traits_type::eof(); - - return traits_type::to_int_type(s[--off]); - } - -}; - -template , class Allocator = std::allocator> -class basic_istringstream_nocopy : public std::basic_iostream -{ - typedef basic_istringbuf_nocopy buf_type; - buf_type buf; -public: - basic_istringstream_nocopy(const typename buf_type::string_type & s) : - std::basic_iostream(&buf), buf(s) {}; -}; - -typedef basic_istringstream_nocopy istringstream_nocopy; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512..8386a4991 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -166,17 +166,30 @@ struct StringSource : Source }; -/* Adapter class of a Source that saves all data read to `s'. */ +/* A sink that writes all incoming data to two other sinks. */ +struct TeeSink : Sink +{ + Sink & sink1, & sink2; + TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } + virtual void operator () (const unsigned char * data, size_t len) + { + sink1(data, len); + sink2(data, len); + } +}; + + +/* Adapter class of a Source that saves all data read to a sink. */ struct TeeSource : Source { Source & orig; - ref data; - TeeSource(Source & orig) - : orig(orig), data(make_ref()) { } + Sink & sink; + TeeSource(Source & orig, Sink & sink) + : orig(orig), sink(sink) { } size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - this->data->append((const char *) data, n); + sink(data, len); return n; } }; @@ -336,4 +349,27 @@ Source & operator >> (Source & in, bool & b) } +/* An adapter that converts a std::basic_istream into a source. */ +struct StreamToSourceAdapter : Source +{ + std::shared_ptr> istream; + + StreamToSourceAdapter(std::shared_ptr> istream) + : istream(istream) + { } + + size_t read(unsigned char * data, size_t len) override + { + if (!istream->read((char *) data, len)) { + if (istream->eof()) { + if (istream->gcount() == 0) + throw EndOfFile("end of file"); + } else + throw Error("I/O error in StreamToSourceAdapter"); + } + return istream->gcount(); + } +}; + + } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..961e7fb6d 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -153,14 +153,15 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + std::optional expectedHash; + Hash hash; std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); if (store->isValidPath(*storePath)) - hash = expectedHash; + hash = *expectedHash; else storePath.reset(); } @@ -200,22 +201,12 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient; addToStore() will also hash - this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error("hash mismatch for '%1%'", uri); - - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); - - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..23bb48d88 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); } @@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - auto paths = store->importPaths(source, nullptr, NoCheckSigs); + auto paths = store->importPaths(source, NoCheckSigs); for (auto & i : paths) cout << fmt("%s\n", store->printStorePath(i)) << std::flush; @@ -878,7 +878,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? + store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index adfd14a75..c97a0bdcf 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -289,8 +289,8 @@ struct InstallableStorePath : Installable ref store; StorePath storePath; - InstallableStorePath(ref store, const Path & storePath) - : store(store), storePath(store->parseStorePath(storePath)) { } + InstallableStorePath(ref store, StorePath && storePath) + : store(store), storePath(std::move(storePath)) { } std::string what() override { return store->printStorePath(storePath); } @@ -595,9 +595,9 @@ std::vector> SourceExprCommand::parseInstallables( if (s.find('/') != std::string::npos) { try { - result.push_back(std::make_shared(store, store->printStorePath(store->followLinksToStorePath(s)))); + result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); continue; - } catch (NotInStore &) { + } catch (BadStorePath &) { } catch (...) { if (!ex) ex = std::current_exception(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bb5e4529b..ce90b0f6d 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -77,13 +77,16 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); - MaintainCount> mcActive(active); update(); auto info = store->queryPathInfo(store->parseStorePath(storePath)); + // Note: info->path can be different from storePath + // for binary cache stores when using --all (since we + // can't enumerate names efficiently). + Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path))); + if (!noContents) { std::unique_ptr hashSink; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 17b63d978..40f1a4f76 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -182,3 +182,56 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" fi # HAVE_LIBSODIUM + + +unset _NIX_FORCE_HTTP + + +# Test 'nix verify --all' on a binary cache. +nix verify -vvvvv --all --store file://$cacheDir --no-trust + + +# Test local NAR caching. +narCache=$TEST_ROOT/nar-cache +rm -rf $narCache +mkdir $narCache + +[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] + +rm -rfv "$cacheDir/nar" + +[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] + +(! nix cat-store --store file://$cacheDir $outPath/foobar) + + +# Test NAR listing generation. +clearCache + +outPath=$(nix-build --no-out-link -E ' + with import ./config.nix; + mkDerivation { + name = "nar-listing"; + buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link"; + } +') + +nix copy --to file://$cacheDir?write-nar-listing=1 $outPath + +[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]] + + +# Test debug info index generation. +clearCache + +outPath=$(nix-build --no-out-link -E ' + with import ./config.nix; + mkDerivation { + name = "debug-info"; + buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"; + } +') + +nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath + +[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]