Add NAR / Store accessor abstraction

This is primary to allow hydra-queue-runner to extract files like
"nix-support/hydra-build-products" from NARs in binary caches.

src/libstore/binary-cache-store.cc

@@ -1,12 +1,13 @@
-#include "binary-cache-store.hh"
-#include "sync.hh"
-
 #include "archive.hh"
+#include "binary-cache-store.hh"
 #include "compression.hh"
 #include "derivations.hh"
+#include "fs-accessor.hh"
 #include "globals.hh"
 #include "nar-info.hh"
+#include "sync.hh"
 #include "worker-protocol.hh"
+#include "nar-accessor.hh"
 
 #include <chrono>
 
@@ -122,7 +123,8 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 
     auto narInfoFile = narInfoFileFor(storePath);
     auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
-    assert(narInfo->path == storePath);
+    if (narInfo->path != storePath)
+        throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
 
     stats.narInfoRead++;
 
@@ -142,6 +144,9 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 
 bool BinaryCacheStore::isValidPath(const Path & storePath)
 {
+    // FIXME: this only checks whether a .narinfo with a matching hash
+    // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
+    // though they shouldn't. Not easily fixed.
     return fileExists(narInfoFileFor(storePath));
 }
 
@@ -344,4 +349,71 @@ void BinaryCacheStore::ensurePath(const Path & path)
     buildPaths({path});
 }
 
+/* Given requests for a path /nix/store/<x>/<y>, this accessor will
+   first download the NAR for /nix/store/<x> from the binary cache,
+   build a NAR accessor for that NAR, and use that to access <y>. */
+struct BinaryCacheStoreAccessor : public FSAccessor
+{
+    ref<BinaryCacheStore> store;
+
+    std::map<Path, ref<FSAccessor>> nars;
+
+    BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
+        : store(store)
+    {
+    }
+
+    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
+    {
+        auto path = canonPath(path_);
+
+        auto storePath = toStorePath(path);
+        std::string restPath = std::string(path, storePath.size());
+
+        if (!store->isValidPath(storePath))
+            throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
+
+        auto i = nars.find(storePath);
+        if (i != nars.end()) return {i->second, restPath};
+
+        StringSink sink;
+        store->exportPath(storePath, false, sink);
+
+        // FIXME: gratuitous string copying.
+        auto accessor = makeNarAccessor(make_ref<std::string>(sink.s));
+        nars.emplace(storePath, accessor);
+        return {accessor, restPath};
+    }
+
+    Stat stat(const Path & path) override
+    {
+        auto res = fetch(path);
+        return res.first->stat(res.second);
+    }
+
+    StringSet readDirectory(const Path & path) override
+    {
+        auto res = fetch(path);
+        return res.first->readDirectory(res.second);
+    }
+
+    std::string readFile(const Path & path) override
+    {
+        auto res = fetch(path);
+        return res.first->readFile(res.second);
+    }
+
+    std::string readLink(const Path & path) override
+    {
+        auto res = fetch(path);
+        return res.first->readLink(res.second);
+    }
+};
+
+ref<FSAccessor> BinaryCacheStore::getFSAccessor()
+{
+    return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>(
+            std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
+}
+
 }

src/libstore/binary-cache-store.hh

@@ -125,8 +125,7 @@ public:
     Path addTextToStore(const string & name, const string & s,
         const PathSet & references, bool repair = false) override;
 
-    void exportPath(const Path & path, bool sign,
+    void exportPath(const Path & path, bool sign, Sink & sink) override;
-        Sink & sink) override;
 
     Paths importPaths(bool requireSignature, Source & source) override;
 
@@ -167,6 +166,8 @@ public:
     bool verifyStore(bool checkContents, bool repair) override
     { return true; }
 
+    ref<FSAccessor> getFSAccessor() override;
+
 };
 
 }

src/libstore/fs-accessor.hh

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+/* An abstract class for accessing a filesystem-like structure, such
+   as a (possibly remote) Nix store or the contents of a NAR file. */
+class FSAccessor
+{
+public:
+    enum Type { tMissing, tRegular, tSymlink, tDirectory };
+
+    struct Stat
+    {
+        Type type;
+        uint64_t fileSize; // regular files only
+        bool isExecutable; // regular files only
+    };
+
+    virtual Stat stat(const Path & path) = 0;
+
+    virtual StringSet readDirectory(const Path & path) = 0;
+
+    virtual std::string readFile(const Path & path) = 0;
+
+    virtual std::string readLink(const Path & path) = 0;
+};
+
+}

src/libstore/local-fs-store.cc

@@ -0,0 +1,71 @@
+#include "fs-accessor.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct LocalStoreAccessor : public FSAccessor
+{
+    ref<Store> store;
+
+    LocalStoreAccessor(ref<Store> store) : store(store) { }
+
+    void assertStore(const Path & path)
+    {
+        Path storePath = toStorePath(path);
+        if (!store->isValidPath(storePath))
+            throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
+    }
+
+    FSAccessor::Stat stat(const Path & path) override
+    {
+        assertStore(path);
+
+        struct stat st;
+        if (lstat(path.c_str(), &st)) {
+            if (errno == ENOENT) return {Type::tMissing, 0, false};
+            throw SysError(format("getting status of ‘%1%’") % path);
+        }
+
+        if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
+            throw Error(format("file ‘%1%’ has unsupported type") % path);
+
+        return {
+            S_ISREG(st.st_mode) ? Type::tRegular :
+            S_ISLNK(st.st_mode) ? Type::tSymlink :
+            Type::tDirectory,
+            S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
+            S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
+    }
+
+    StringSet readDirectory(const Path & path) override
+    {
+        assertStore(path);
+
+        auto entries = nix::readDirectory(path);
+
+        StringSet res;
+        for (auto & entry : entries)
+            res.insert(entry.name);
+
+        return res;
+    }
+
+    std::string readFile(const Path & path) override
+    {
+        assertStore(path);
+        return nix::readFile(path);
+    }
+
+    std::string readLink(const Path & path) override
+    {
+        assertStore(path);
+        return nix::readLink(path);
+    }
+};
+
+ref<FSAccessor> LocalFSStore::getFSAccessor()
+{
+    return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this()));
+}
+
+}

src/libstore/local-store.hh

@@ -80,7 +80,7 @@ struct SQLiteStmt
 };
 
 
-class LocalStore : public Store
+class LocalStore : public LocalFSStore
 {
 private:
     typedef std::map<Path, RunningSubstituter> RunningSubstituters;
@@ -170,14 +170,11 @@ public:
        files with the same contents. */
     void optimiseStore(OptimiseStats & stats);
 
-    /* Generic variant of the above method.  */
     void optimiseStore() override;
 
     /* Optimise a single store path. */
     void optimisePath(const Path & path);
 
-    /* Check the integrity of the Nix store.  Returns true if errors
-       remain. */
     bool verifyStore(bool checkContents, bool repair) override;
 
     /* Register the validity of a path, i.e., that `path' exists, that

src/libstore/nar-accessor.cc

@@ -0,0 +1,142 @@
+#include "nar-accessor.hh"
+#include "archive.hh"
+
+#include <map>
+
+namespace nix {
+
+struct NarMember
+{
+    FSAccessor::Type type;
+
+    bool isExecutable;
+
+    /* If this is a regular file, position of the contents of this
+       file in the NAR. */
+    size_t start, size;
+
+    std::string target;
+};
+
+struct NarIndexer : ParseSink, StringSource
+{
+    // FIXME: should store this as a tree. Now we're vulnerable to
+    // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
+    typedef std::map<Path, NarMember> Members;
+    Members members;
+
+    Path currentPath;
+    std::string currentStart;
+    bool isExec;
+
+    NarIndexer(const std::string & nar) : StringSource(nar)
+    {
+    }
+
+    void createDirectory(const Path & path)
+    {
+        members.emplace(path,
+            NarMember{FSAccessor::Type::tDirectory, false, 0, 0});
+    }
+
+    void createRegularFile(const Path & path) override
+    {
+        currentPath = path;
+    }
+
+    void isExecutable()
+    {
+        isExec = true;
+    }
+
+    void preallocateContents(unsigned long long size) override
+    {
+        assert(currentPath != "");
+        currentStart = string(s, pos, 16);
+        members.emplace(currentPath,
+            NarMember{FSAccessor::Type::tRegular, isExec, pos, size});
+    }
+
+    void receiveContents(unsigned char * data, unsigned int len) override
+    {
+        // Sanity check
+        if (!currentStart.empty()) {
+            assert(len < 16 || currentStart == string((char *) data, 16));
+            currentStart.clear();
+        }
+    }
+
+    void createSymlink(const Path & path, const string & target) override
+    {
+        members.emplace(path,
+            NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
+    }
+
+    Members::iterator find(const Path & path)
+    {
+        auto i = members.find(path);
+        if (i == members.end())
+            throw Error(format("NAR file does not contain path ‘%1%’") % path);
+        return i;
+    }
+};
+
+struct NarAccessor : public FSAccessor
+{
+    ref<const std::string> nar;
+    NarIndexer indexer;
+
+    NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
+    {
+        parseDump(indexer, indexer);
+    }
+
+    Stat stat(const Path & path) override
+    {
+        auto i = indexer.members.find(path);
+        if (i == indexer.members.end())
+            return {FSAccessor::Type::tMissing, 0, false};
+        return {i->second.type, i->second.size, i->second.isExecutable};
+    }
+
+    StringSet readDirectory(const Path & path) override
+    {
+        auto i = indexer.find(path);
+
+        if (i->second.type != FSAccessor::Type::tDirectory)
+            throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
+
+        ++i;
+        StringSet res;
+        while (i != indexer.members.end() && isInDir(i->first, path)) {
+            // FIXME: really bad performance.
+            if (i->first.find('/', path.size() + 1) == std::string::npos)
+                res.insert(std::string(i->first, path.size() + 1));
+            ++i;
+        }
+        return res;
+    }
+
+    std::string readFile(const Path & path) override
+    {
+        auto i = indexer.find(path);
+        if (i->second.type != FSAccessor::Type::tRegular)
+            throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
+        return std::string(*nar, i->second.start, i->second.size);
+    }
+
+    std::string readLink(const Path & path) override
+    {
+        auto i = indexer.find(path);
+        if (i->second.type != FSAccessor::Type::tSymlink)
+            throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
+        return i->second.target;
+    }
+};
+
+ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
+{
+    return make_ref<NarAccessor>(nar);
+}
+
+}

src/libstore/nar-accessor.hh

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "fs-accessor.hh"
+
+namespace nix {
+
+/* Return an object that provides access to the contents of a NAR
+   file. */
+ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
+
+}

src/libstore/remote-store.hh

@@ -16,7 +16,9 @@ struct FdSource;
 template<typename T> class Pool;
 
 
-class RemoteStore : public Store
+/* FIXME: RemoteStore is a misnomer - should be something like
+   DaemonStore. */
+class RemoteStore : public LocalFSStore
 {
 public:
 

src/libstore/store-api.cc

@@ -332,7 +332,9 @@ ref<Store> openStoreAt(const std::string & uri)
 
     enum { mDaemon, mLocal, mAuto } mode;
 
-    mode = uri == "daemon" ? mDaemon : mAuto;
+    mode =
+        uri == "daemon" ? mDaemon :
+        uri == "local" ? mLocal : mAuto;
 
     if (mode == mAuto) {
         if (LocalStore::haveWriteAccess())

src/libstore/store-api.hh

@@ -140,9 +140,10 @@ struct BuildResult
 
 struct BasicDerivation;
 struct Derivation;
+struct FSAccessor;
 
 
-class Store
+class Store : public std::enable_shared_from_this<Store>
 {
 public:
 
@@ -314,6 +315,9 @@ public:
        remain. */
     virtual bool verifyStore(bool checkContents, bool repair) = 0;
 
+    /* Return an object to access files in the Nix store. */
+    virtual ref<FSAccessor> getFSAccessor() = 0;
+
     /* Utility functions. */
 
     /* Read a derivation, after ensuring its existence through
@@ -345,6 +349,12 @@ public:
 };
 
 
+class LocalFSStore : public Store
+{
+    ref<FSAccessor> getFSAccessor() override;
+};
+
+
 /* !!! These should be part of the store API, I guess. */
 
 /* Throw an exception if `path' is not directly in the Nix store. */

src/libutil/archive.hh

@@ -57,6 +57,7 @@ void dumpPath(const Path & path, Sink & sink,
 
 void dumpString(const std::string & s, Sink & sink);
 
+/* FIXME: fix this API, it sucks. */
 struct ParseSink
 {
     virtual void createDirectory(const Path & path) { };