From cca4a8dc1a622ab086639e5c09347303c062922e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 15:20:10 +0100 Subject: [PATCH] importPaths(): Optionally add NARs to binary cache accessor This enables an optimisation in hydra-queue-runner, preventing a download of a NAR it just uploaded to the cache when reading files like hydra-build-products. --- perl/lib/Nix/Store.xs | 2 +- src/libstore/binary-cache-store.cc | 66 ++++++++++++++++-------------- src/libstore/binary-cache-store.hh | 5 ++- src/libstore/local-store.cc | 3 +- src/libstore/local-store.hh | 3 +- src/libstore/remote-store.cc | 3 +- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.hh | 7 +++- src/nix-daemon/nix-daemon.cc | 6 +-- src/nix-store/nix-store.cc | 4 +- 10 files changed, 58 insertions(+), 44 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 44c88a87b..bb322875d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -186,7 +186,7 @@ void importPaths(int fd) PPCODE: try { FdSource source(fd); - store()->importPaths(false, source); + store()->importPaths(false, source, 0); } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a4e3a560..02e73d2ce 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink sink << exportMagic << storePath << res.references << res.deriver << 0; } -Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { assert(!requireSignature); Paths res; @@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) unsigned long long n = readLongLong(source); if (n == 0) break; if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(source)); + res.push_back(importPath(source, accessor)); } return res; } @@ -215,34 +216,6 @@ struct NopSink : ParseSink { }; -Path BinaryCacheStore::importPath(Source & source) -{ - /* FIXME: some cut&paste of LocalStore::importPath(). */ - - /* Extract the NAR from the source. */ - TeeSource tee(source); - NopSink sink; - parseDump(sink, tee); - - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); - - ValidPathInfo info; - info.path = readStorePath(source); - - info.references = readStorePaths(source); - - readString(source); // deriver, don't care - - bool haveSignature = readInt(source) == 1; - assert(!haveSignature); - - addToCache(info, tee.data); - - return info.path; -} - ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) { return ValidPathInfo(readNarInfo(storePath)); @@ -416,4 +389,37 @@ ref BinaryCacheStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } +Path BinaryCacheStore::importPath(Source & source, std::shared_ptr accessor) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, tee.data); + + auto accessor_ = std::dynamic_pointer_cast(accessor); + if (accessor_) + // FIXME: more gratuitous string copying + accessor_->nars.emplace(info.path, makeNarAccessor(make_ref(tee.data))); + + return info.path; +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c6f319d1b..6feb84cd2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -127,9 +127,10 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; - Path importPath(Source & source); + Path importPath(Source & source, std::shared_ptr accessor); void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 33f256912..9a5706681 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -Paths LocalStore::importPaths(bool requireSignature, Source & source) +Paths LocalStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { Paths res; while (true) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fded63ab3..c7ea9e503 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -147,7 +147,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a1ac169c2..82b7cfd7c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Paths RemoteStore::importPaths(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { auto conn(connections->get()); conn->to << wopImportPaths; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0019cd8f9..85c8292c7 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 347387f03..488f32e16 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -227,8 +227,11 @@ public: void exportPaths(const Paths & paths, bool sign, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + virtual Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index e3c12ea0e..5189c9b4c 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -310,7 +310,7 @@ static void performOp(ref store, bool trusted, unsigned int clientVe case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(!trusted, source); + Paths paths = store->importPaths(!trusted, source, 0); stopWork(); to << paths; break; @@ -322,8 +322,8 @@ static void performOp(ref store, bool trusted, unsigned int clientVe if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode)readInt(from); - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ if (mode == bmRepair && !trusted) throw Error("repairing is not supported when building through the Nix daemon"); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 48c2865e5..a258c77c3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(requireSignature, source); + Paths paths = store->importPaths(requireSignature, source, 0); for (auto & i : paths) cout << format("%1%\n") % i << std::flush; @@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(false, in); + store->importPaths(false, in, 0); out << 1; // indicate success break; }