importPaths(): Optionally add NARs to binary cache accessor

This enables an optimisation in hydra-queue-runner, preventing a
download of a NAR it just uploaded to the cache when reading files
like hydra-build-products.
This commit is contained in:
Eelco Dolstra 2016-02-26 15:20:10 +01:00
parent 00b2c05749
commit cca4a8dc1a
10 changed files with 58 additions and 44 deletions

View file

@ -186,7 +186,7 @@ void importPaths(int fd)
PPCODE: PPCODE:
try { try {
FdSource source(fd); FdSource source(fd);
store()->importPaths(false, source); store()->importPaths(false, source, 0);
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
} }

View file

@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
sink << exportMagic << storePath << res.references << res.deriver << 0; sink << exportMagic << storePath << res.references << res.deriver << 0;
} }
Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{ {
assert(!requireSignature); assert(!requireSignature);
Paths res; Paths res;
@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
unsigned long long n = readLongLong(source); unsigned long long n = readLongLong(source);
if (n == 0) break; if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by nix-store --export"); if (n != 1) throw Error("input doesn't look like something created by nix-store --export");
res.push_back(importPath(source)); res.push_back(importPath(source, accessor));
} }
return res; return res;
} }
@ -215,34 +216,6 @@ struct NopSink : ParseSink
{ {
}; };
Path BinaryCacheStore::importPath(Source & source)
{
/* FIXME: some cut&paste of LocalStore::importPath(). */
/* Extract the NAR from the source. */
TeeSource tee(source);
NopSink sink;
parseDump(sink, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info;
info.path = readStorePath(source);
info.references = readStorePaths<PathSet>(source);
readString(source); // deriver, don't care
bool haveSignature = readInt(source) == 1;
assert(!haveSignature);
addToCache(info, tee.data);
return info.path;
}
ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
{ {
return ValidPathInfo(readNarInfo(storePath)); return ValidPathInfo(readNarInfo(storePath));
@ -416,4 +389,37 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
} }
Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor)
{
/* FIXME: some cut&paste of LocalStore::importPath(). */
/* Extract the NAR from the source. */
TeeSource tee(source);
NopSink sink;
parseDump(sink, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info;
info.path = readStorePath(source);
info.references = readStorePaths<PathSet>(source);
readString(source); // deriver, don't care
bool haveSignature = readInt(source) == 1;
assert(!haveSignature);
addToCache(info, tee.data);
auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
if (accessor_)
// FIXME: more gratuitous string copying
accessor_->nars.emplace(info.path, makeNarAccessor(make_ref<std::string>(tee.data)));
return info.path;
}
} }

View file

@ -127,9 +127,10 @@ public:
void exportPath(const Path & path, bool sign, Sink & sink) override; void exportPath(const Path & path, bool sign, Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override; Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
Path importPath(Source & source); Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor);
void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;

View file

@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
} }
Paths LocalStore::importPaths(bool requireSignature, Source & source) Paths LocalStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{ {
Paths res; Paths res;
while (true) { while (true) {

View file

@ -147,7 +147,8 @@ public:
void exportPath(const Path & path, bool sign, void exportPath(const Path & path, bool sign,
Sink & sink) override; Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override; Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override;

View file

@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign,
} }
Paths RemoteStore::importPaths(bool requireSignature, Source & source) Paths RemoteStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{ {
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopImportPaths; conn->to << wopImportPaths;

View file

@ -65,7 +65,8 @@ public:
void exportPath(const Path & path, bool sign, void exportPath(const Path & path, bool sign,
Sink & sink) override; Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override; Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override;

View file

@ -227,8 +227,11 @@ public:
void exportPaths(const Paths & paths, bool sign, Sink & sink); void exportPaths(const Paths & paths, bool sign, Sink & sink);
/* Import a sequence of NAR dumps created by exportPaths() into /* Import a sequence of NAR dumps created by exportPaths() into
the Nix store. */ the Nix store. Optionally, the contents of the NARs are
virtual Paths importPaths(bool requireSignature, Source & source) = 0; preloaded into the specified FS accessor to speed up subsequent
access. */
virtual Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) = 0;
/* For each path, if it's a derivation, build it. Building a /* For each path, if it's a derivation, build it. Building a
derivation means ensuring that the output paths are valid. If derivation means ensuring that the output paths are valid. If

View file

@ -310,7 +310,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
case wopImportPaths: { case wopImportPaths: {
startWork(); startWork();
TunnelSource source(from); TunnelSource source(from);
Paths paths = store->importPaths(!trusted, source); Paths paths = store->importPaths(!trusted, source, 0);
stopWork(); stopWork();
to << paths; to << paths;
break; break;
@ -322,8 +322,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode)readInt(from); mode = (BuildMode)readInt(from);
/* Repairing is not atomic, so disallowed for "untrusted" /* Repairing is not atomic, so disallowed for "untrusted"
clients. */ clients. */
if (mode == bmRepair && !trusted) if (mode == bmRepair && !trusted)
throw Error("repairing is not supported when building through the Nix daemon"); throw Error("repairing is not supported when building through the Nix daemon");
} }

View file

@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected"); if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO); FdSource source(STDIN_FILENO);
Paths paths = store->importPaths(requireSignature, source); Paths paths = store->importPaths(requireSignature, source, 0);
for (auto & i : paths) for (auto & i : paths)
cout << format("%1%\n") % i << std::flush; cout << format("%1%\n") % i << std::flush;
@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: { case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed"); if (!writeAllowed) throw Error("importing paths is not allowed");
store->importPaths(false, in); store->importPaths(false, in, 0);
out << 1; // indicate success out << 1; // indicate success
break; break;
} }