From 1042c10fd0417fe33dd879317f5d7a73aa6f7fe3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2016 17:43:19 +0100 Subject: [PATCH] Add NAR / Store accessor abstraction This is primary to allow hydra-queue-runner to extract files like "nix-support/hydra-build-products" from NARs in binary caches. --- src/libstore/binary-cache-store.cc | 80 +++++++++++++++- src/libstore/binary-cache-store.hh | 5 +- src/libstore/fs-accessor.hh | 30 ++++++ src/libstore/local-fs-store.cc | 71 +++++++++++++++ src/libstore/local-store.hh | 5 +- src/libstore/nar-accessor.cc | 142 +++++++++++++++++++++++++++++ src/libstore/nar-accessor.hh | 11 +++ src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 4 +- src/libstore/store-api.hh | 12 ++- src/libutil/archive.hh | 1 + 11 files changed, 352 insertions(+), 13 deletions(-) create mode 100644 src/libstore/fs-accessor.hh create mode 100644 src/libstore/local-fs-store.cc create mode 100644 src/libstore/nar-accessor.cc create mode 100644 src/libstore/nar-accessor.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index dc086fe9c..6a4e3a560 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -1,12 +1,13 @@ -#include "binary-cache-store.hh" -#include "sync.hh" - #include "archive.hh" +#include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" +#include "fs-accessor.hh" #include "globals.hh" #include "nar-info.hh" +#include "sync.hh" #include "worker-protocol.hh" +#include "nar-accessor.hh" #include @@ -122,7 +123,8 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) auto narInfoFile = narInfoFileFor(storePath); auto narInfo = make_ref(getFile(narInfoFile), narInfoFile); - assert(narInfo->path == storePath); + if (narInfo->path != storePath) + throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath); stats.narInfoRead++; @@ -142,6 +144,9 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) bool BinaryCacheStore::isValidPath(const Path & storePath) { + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. return fileExists(narInfoFileFor(storePath)); } @@ -344,4 +349,71 @@ void BinaryCacheStore::ensurePath(const Path & path) buildPaths({path}); } +/* Given requests for a path /nix/store//, this accessor will + first download the NAR for /nix/store/ from the binary cache, + build a NAR accessor for that NAR, and use that to access . */ +struct BinaryCacheStoreAccessor : public FSAccessor +{ + ref store; + + std::map> nars; + + BinaryCacheStoreAccessor(ref store) + : store(store) + { + } + + std::pair, Path> fetch(const Path & path_) + { + auto path = canonPath(path_); + + auto storePath = toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->exportPath(storePath, false, sink); + + // FIXME: gratuitous string copying. + auto accessor = makeNarAccessor(make_ref(sink.s)); + nars.emplace(storePath, accessor); + return {accessor, restPath}; + } + + Stat stat(const Path & path) override + { + auto res = fetch(path); + return res.first->stat(res.second); + } + + StringSet readDirectory(const Path & path) override + { + auto res = fetch(path); + return res.first->readDirectory(res.second); + } + + std::string readFile(const Path & path) override + { + auto res = fetch(path); + return res.first->readFile(res.second); + } + + std::string readLink(const Path & path) override + { + auto res = fetch(path); + return res.first->readLink(res.second); + } +}; + +ref BinaryCacheStore::getFSAccessor() +{ + return make_ref(ref( + std::dynamic_pointer_cast(shared_from_this()))); +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 2235d6d67..c6f319d1b 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -125,8 +125,7 @@ public: Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) override; - void exportPath(const Path & path, bool sign, - Sink & sink) override; + void exportPath(const Path & path, bool sign, Sink & sink) override; Paths importPaths(bool requireSignature, Source & source) override; @@ -167,6 +166,8 @@ public: bool verifyStore(bool checkContents, bool repair) override { return true; } + ref getFSAccessor() override; + }; } diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh new file mode 100644 index 000000000..a67e0775b --- /dev/null +++ b/src/libstore/fs-accessor.hh @@ -0,0 +1,30 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* An abstract class for accessing a filesystem-like structure, such + as a (possibly remote) Nix store or the contents of a NAR file. */ +class FSAccessor +{ +public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; + + struct Stat + { + Type type; + uint64_t fileSize; // regular files only + bool isExecutable; // regular files only + }; + + virtual Stat stat(const Path & path) = 0; + + virtual StringSet readDirectory(const Path & path) = 0; + + virtual std::string readFile(const Path & path) = 0; + + virtual std::string readLink(const Path & path) = 0; +}; + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc new file mode 100644 index 000000000..7094a50a3 --- /dev/null +++ b/src/libstore/local-fs-store.cc @@ -0,0 +1,71 @@ +#include "fs-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct LocalStoreAccessor : public FSAccessor +{ + ref store; + + LocalStoreAccessor(ref store) : store(store) { } + + void assertStore(const Path & path) + { + Path storePath = toStorePath(path); + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + } + + FSAccessor::Stat stat(const Path & path) override + { + assertStore(path); + + struct stat st; + if (lstat(path.c_str(), &st)) { + if (errno == ENOENT) return {Type::tMissing, 0, false}; + throw SysError(format("getting status of ‘%1%’") % path); + } + + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + throw Error(format("file ‘%1%’ has unsupported type") % path); + + return { + S_ISREG(st.st_mode) ? Type::tRegular : + S_ISLNK(st.st_mode) ? Type::tSymlink : + Type::tDirectory, + S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + } + + StringSet readDirectory(const Path & path) override + { + assertStore(path); + + auto entries = nix::readDirectory(path); + + StringSet res; + for (auto & entry : entries) + res.insert(entry.name); + + return res; + } + + std::string readFile(const Path & path) override + { + assertStore(path); + return nix::readFile(path); + } + + std::string readLink(const Path & path) override + { + assertStore(path); + return nix::readLink(path); + } +}; + +ref LocalFSStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this())); +} + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5582acd0f..fded63ab3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -80,7 +80,7 @@ struct SQLiteStmt }; -class LocalStore : public Store +class LocalStore : public LocalFSStore { private: typedef std::map RunningSubstituters; @@ -170,14 +170,11 @@ public: files with the same contents. */ void optimiseStore(OptimiseStats & stats); - /* Generic variant of the above method. */ void optimiseStore() override; /* Optimise a single store path. */ void optimisePath(const Path & path); - /* Check the integrity of the Nix store. Returns true if errors - remain. */ bool verifyStore(bool checkContents, bool repair) override; /* Register the validity of a path, i.e., that `path' exists, that diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc new file mode 100644 index 000000000..88b027ff3 --- /dev/null +++ b/src/libstore/nar-accessor.cc @@ -0,0 +1,142 @@ +#include "nar-accessor.hh" +#include "archive.hh" + +#include + +namespace nix { + +struct NarMember +{ + FSAccessor::Type type; + + bool isExecutable; + + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start, size; + + std::string target; +}; + +struct NarIndexer : ParseSink, StringSource +{ + // FIXME: should store this as a tree. Now we're vulnerable to + // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). + typedef std::map Members; + Members members; + + Path currentPath; + std::string currentStart; + bool isExec; + + NarIndexer(const std::string & nar) : StringSource(nar) + { + } + + void createDirectory(const Path & path) + { + members.emplace(path, + NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path & path) override + { + currentPath = path; + } + + void isExecutable() + { + isExec = true; + } + + void preallocateContents(unsigned long long size) override + { + assert(currentPath != ""); + currentStart = string(s, pos, 16); + members.emplace(currentPath, + NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char *) data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + members.emplace(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + + Members::iterator find(const Path & path) + { + auto i = members.find(path); + if (i == members.end()) + throw Error(format("NAR file does not contain path ‘%1%’") % path); + return i; + } +}; + +struct NarAccessor : public FSAccessor +{ + ref nar; + NarIndexer indexer; + + NarAccessor(ref nar) : nar(nar), indexer(*nar) + { + parseDump(indexer, indexer); + } + + Stat stat(const Path & path) override + { + auto i = indexer.members.find(path); + if (i == indexer.members.end()) + return {FSAccessor::Type::tMissing, 0, false}; + return {i->second.type, i->second.size, i->second.isExecutable}; + } + + StringSet readDirectory(const Path & path) override + { + auto i = indexer.find(path); + + if (i->second.type != FSAccessor::Type::tDirectory) + throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); + + ++i; + StringSet res; + while (i != indexer.members.end() && isInDir(i->first, path)) { + // FIXME: really bad performance. + if (i->first.find('/', path.size() + 1) == std::string::npos) + res.insert(std::string(i->first, path.size() + 1)); + ++i; + } + return res; + } + + std::string readFile(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); + return std::string(*nar, i->second.start, i->second.size); + } + + std::string readLink(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tSymlink) + throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); + return i->second.target; + } +}; + +ref makeNarAccessor(ref nar) +{ + return make_ref(nar); +} + +} diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh new file mode 100644 index 000000000..83c570be4 --- /dev/null +++ b/src/libstore/nar-accessor.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "fs-accessor.hh" + +namespace nix { + +/* Return an object that provides access to the contents of a NAR + file. */ +ref makeNarAccessor(ref nar); + +} diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ddfb70a66..0019cd8f9 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -16,7 +16,9 @@ struct FdSource; template class Pool; -class RemoteStore : public Store +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public LocalFSStore { public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7058249f0..24c05b8b4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -332,7 +332,9 @@ ref openStoreAt(const std::string & uri) enum { mDaemon, mLocal, mAuto } mode; - mode = uri == "daemon" ? mDaemon : mAuto; + mode = + uri == "daemon" ? mDaemon : + uri == "local" ? mLocal : mAuto; if (mode == mAuto) { if (LocalStore::haveWriteAccess()) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 84ede157e..347387f03 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -140,9 +140,10 @@ struct BuildResult struct BasicDerivation; struct Derivation; +struct FSAccessor; -class Store +class Store : public std::enable_shared_from_this { public: @@ -314,6 +315,9 @@ public: remain. */ virtual bool verifyStore(bool checkContents, bool repair) = 0; + /* Return an object to access files in the Nix store. */ + virtual ref getFSAccessor() = 0; + /* Utility functions. */ /* Read a derivation, after ensuring its existence through @@ -345,6 +349,12 @@ public: }; +class LocalFSStore : public Store +{ + ref getFSAccessor() override; +}; + + /* !!! These should be part of the store API, I guess. */ /* Throw an exception if `path' is not directly in the Nix store. */ diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 90117f5ff..d58b91df0 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -57,6 +57,7 @@ void dumpPath(const Path & path, Sink & sink, void dumpString(const std::string & s, Sink & sink); +/* FIXME: fix this API, it sucks. */ struct ParseSink { virtual void createDirectory(const Path & path) { };