Add NAR / Store accessor abstraction

This is primary to allow hydra-queue-runner to extract files like
"nix-support/hydra-build-products" from NARs in binary caches.
This commit is contained in:
Eelco Dolstra 2016-02-25 17:43:19 +01:00
parent 152b1d6bf9
commit 1042c10fd0
11 changed files with 352 additions and 13 deletions

View file

@ -1,12 +1,13 @@
#include "binary-cache-store.hh"
#include "sync.hh"
#include "archive.hh" #include "archive.hh"
#include "binary-cache-store.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "sync.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "nar-accessor.hh"
#include <chrono> #include <chrono>
@ -122,7 +123,8 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
auto narInfoFile = narInfoFileFor(storePath); auto narInfoFile = narInfoFileFor(storePath);
auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile); auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
assert(narInfo->path == storePath); if (narInfo->path != storePath)
throw Error(format("NAR info file for store path %1% does not match %2%") % narInfo->path % storePath);
stats.narInfoRead++; stats.narInfoRead++;
@ -142,6 +144,9 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
bool BinaryCacheStore::isValidPath(const Path & storePath) bool BinaryCacheStore::isValidPath(const Path & storePath)
{ {
// FIXME: this only checks whether a .narinfo with a matching hash
// part exists. So f4kb...-foo matches f4kb...-bar, even
// though they shouldn't. Not easily fixed.
return fileExists(narInfoFileFor(storePath)); return fileExists(narInfoFileFor(storePath));
} }
@ -344,4 +349,71 @@ void BinaryCacheStore::ensurePath(const Path & path)
buildPaths({path}); buildPaths({path});
} }
/* Given requests for a path /nix/store/<x>/<y>, this accessor will
first download the NAR for /nix/store/<x> from the binary cache,
build a NAR accessor for that NAR, and use that to access <y>. */
struct BinaryCacheStoreAccessor : public FSAccessor
{
ref<BinaryCacheStore> store;
std::map<Path, ref<FSAccessor>> nars;
BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
: store(store)
{
}
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
throw Error(format("path %1% is not a valid store path") % storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
store->exportPath(storePath, false, sink);
// FIXME: gratuitous string copying.
auto accessor = makeNarAccessor(make_ref<std::string>(sink.s));
nars.emplace(storePath, accessor);
return {accessor, restPath};
}
Stat stat(const Path & path) override
{
auto res = fetch(path);
return res.first->stat(res.second);
}
StringSet readDirectory(const Path & path) override
{
auto res = fetch(path);
return res.first->readDirectory(res.second);
}
std::string readFile(const Path & path) override
{
auto res = fetch(path);
return res.first->readFile(res.second);
}
std::string readLink(const Path & path) override
{
auto res = fetch(path);
return res.first->readLink(res.second);
}
};
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{
return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>(
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
}
} }

View file

@ -125,8 +125,7 @@ public:
Path addTextToStore(const string & name, const string & s, Path addTextToStore(const string & name, const string & s,
const PathSet & references, bool repair = false) override; const PathSet & references, bool repair = false) override;
void exportPath(const Path & path, bool sign, void exportPath(const Path & path, bool sign, Sink & sink) override;
Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override; Paths importPaths(bool requireSignature, Source & source) override;
@ -167,6 +166,8 @@ public:
bool verifyStore(bool checkContents, bool repair) override bool verifyStore(bool checkContents, bool repair) override
{ return true; } { return true; }
ref<FSAccessor> getFSAccessor() override;
}; };
} }

View file

@ -0,0 +1,30 @@
#pragma once
#include "types.hh"
namespace nix {
/* An abstract class for accessing a filesystem-like structure, such
as a (possibly remote) Nix store or the contents of a NAR file. */
class FSAccessor
{
public:
enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat
{
Type type;
uint64_t fileSize; // regular files only
bool isExecutable; // regular files only
};
virtual Stat stat(const Path & path) = 0;
virtual StringSet readDirectory(const Path & path) = 0;
virtual std::string readFile(const Path & path) = 0;
virtual std::string readLink(const Path & path) = 0;
};
}

View file

@ -0,0 +1,71 @@
#include "fs-accessor.hh"
#include "store-api.hh"
namespace nix {
struct LocalStoreAccessor : public FSAccessor
{
ref<Store> store;
LocalStoreAccessor(ref<Store> store) : store(store) { }
void assertStore(const Path & path)
{
Path storePath = toStorePath(path);
if (!store->isValidPath(storePath))
throw Error(format("path %1% is not a valid store path") % storePath);
}
FSAccessor::Stat stat(const Path & path) override
{
assertStore(path);
struct stat st;
if (lstat(path.c_str(), &st)) {
if (errno == ENOENT) return {Type::tMissing, 0, false};
throw SysError(format("getting status of %1%") % path);
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error(format("file %1% has unsupported type") % path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
}
StringSet readDirectory(const Path & path) override
{
assertStore(path);
auto entries = nix::readDirectory(path);
StringSet res;
for (auto & entry : entries)
res.insert(entry.name);
return res;
}
std::string readFile(const Path & path) override
{
assertStore(path);
return nix::readFile(path);
}
std::string readLink(const Path & path) override
{
assertStore(path);
return nix::readLink(path);
}
};
ref<FSAccessor> LocalFSStore::getFSAccessor()
{
return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this()));
}
}

View file

@ -80,7 +80,7 @@ struct SQLiteStmt
}; };
class LocalStore : public Store class LocalStore : public LocalFSStore
{ {
private: private:
typedef std::map<Path, RunningSubstituter> RunningSubstituters; typedef std::map<Path, RunningSubstituter> RunningSubstituters;
@ -170,14 +170,11 @@ public:
files with the same contents. */ files with the same contents. */
void optimiseStore(OptimiseStats & stats); void optimiseStore(OptimiseStats & stats);
/* Generic variant of the above method. */
void optimiseStore() override; void optimiseStore() override;
/* Optimise a single store path. */ /* Optimise a single store path. */
void optimisePath(const Path & path); void optimisePath(const Path & path);
/* Check the integrity of the Nix store. Returns true if errors
remain. */
bool verifyStore(bool checkContents, bool repair) override; bool verifyStore(bool checkContents, bool repair) override;
/* Register the validity of a path, i.e., that `path' exists, that /* Register the validity of a path, i.e., that `path' exists, that

View file

@ -0,0 +1,142 @@
#include "nar-accessor.hh"
#include "archive.hh"
#include <map>
namespace nix {
struct NarMember
{
FSAccessor::Type type;
bool isExecutable;
/* If this is a regular file, position of the contents of this
file in the NAR. */
size_t start, size;
std::string target;
};
struct NarIndexer : ParseSink, StringSource
{
// FIXME: should store this as a tree. Now we're vulnerable to
// O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
typedef std::map<Path, NarMember> Members;
Members members;
Path currentPath;
std::string currentStart;
bool isExec;
NarIndexer(const std::string & nar) : StringSource(nar)
{
}
void createDirectory(const Path & path)
{
members.emplace(path,
NarMember{FSAccessor::Type::tDirectory, false, 0, 0});
}
void createRegularFile(const Path & path) override
{
currentPath = path;
}
void isExecutable()
{
isExec = true;
}
void preallocateContents(unsigned long long size) override
{
assert(currentPath != "");
currentStart = string(s, pos, 16);
members.emplace(currentPath,
NarMember{FSAccessor::Type::tRegular, isExec, pos, size});
}
void receiveContents(unsigned char * data, unsigned int len) override
{
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
void createSymlink(const Path & path, const string & target) override
{
members.emplace(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
Members::iterator find(const Path & path)
{
auto i = members.find(path);
if (i == members.end())
throw Error(format("NAR file does not contain path %1%") % path);
return i;
}
};
struct NarAccessor : public FSAccessor
{
ref<const std::string> nar;
NarIndexer indexer;
NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
{
parseDump(indexer, indexer);
}
Stat stat(const Path & path) override
{
auto i = indexer.members.find(path);
if (i == indexer.members.end())
return {FSAccessor::Type::tMissing, 0, false};
return {i->second.type, i->second.size, i->second.isExecutable};
}
StringSet readDirectory(const Path & path) override
{
auto i = indexer.find(path);
if (i->second.type != FSAccessor::Type::tDirectory)
throw Error(format("path %1% inside NAR file is not a directory") % path);
++i;
StringSet res;
while (i != indexer.members.end() && isInDir(i->first, path)) {
// FIXME: really bad performance.
if (i->first.find('/', path.size() + 1) == std::string::npos)
res.insert(std::string(i->first, path.size() + 1));
++i;
}
return res;
}
std::string readFile(const Path & path) override
{
auto i = indexer.find(path);
if (i->second.type != FSAccessor::Type::tRegular)
throw Error(format("path %1% inside NAR file is not a regular file") % path);
return std::string(*nar, i->second.start, i->second.size);
}
std::string readLink(const Path & path) override
{
auto i = indexer.find(path);
if (i->second.type != FSAccessor::Type::tSymlink)
throw Error(format("path %1% inside NAR file is not a symlink") % path);
return i->second.target;
}
};
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
{
return make_ref<NarAccessor>(nar);
}
}

View file

@ -0,0 +1,11 @@
#pragma once
#include "fs-accessor.hh"
namespace nix {
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
}

View file

@ -16,7 +16,9 @@ struct FdSource;
template<typename T> class Pool; template<typename T> class Pool;
class RemoteStore : public Store /* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
class RemoteStore : public LocalFSStore
{ {
public: public:

View file

@ -332,7 +332,9 @@ ref<Store> openStoreAt(const std::string & uri)
enum { mDaemon, mLocal, mAuto } mode; enum { mDaemon, mLocal, mAuto } mode;
mode = uri == "daemon" ? mDaemon : mAuto; mode =
uri == "daemon" ? mDaemon :
uri == "local" ? mLocal : mAuto;
if (mode == mAuto) { if (mode == mAuto) {
if (LocalStore::haveWriteAccess()) if (LocalStore::haveWriteAccess())

View file

@ -140,9 +140,10 @@ struct BuildResult
struct BasicDerivation; struct BasicDerivation;
struct Derivation; struct Derivation;
struct FSAccessor;
class Store class Store : public std::enable_shared_from_this<Store>
{ {
public: public:
@ -314,6 +315,9 @@ public:
remain. */ remain. */
virtual bool verifyStore(bool checkContents, bool repair) = 0; virtual bool verifyStore(bool checkContents, bool repair) = 0;
/* Return an object to access files in the Nix store. */
virtual ref<FSAccessor> getFSAccessor() = 0;
/* Utility functions. */ /* Utility functions. */
/* Read a derivation, after ensuring its existence through /* Read a derivation, after ensuring its existence through
@ -345,6 +349,12 @@ public:
}; };
class LocalFSStore : public Store
{
ref<FSAccessor> getFSAccessor() override;
};
/* !!! These should be part of the store API, I guess. */ /* !!! These should be part of the store API, I guess. */
/* Throw an exception if `path' is not directly in the Nix store. */ /* Throw an exception if `path' is not directly in the Nix store. */

View file

@ -57,6 +57,7 @@ void dumpPath(const Path & path, Sink & sink,
void dumpString(const std::string & s, Sink & sink); void dumpString(const std::string & s, Sink & sink);
/* FIXME: fix this API, it sucks. */
struct ParseSink struct ParseSink
{ {
virtual void createDirectory(const Path & path) { }; virtual void createDirectory(const Path & path) { };