forked from lix-project/lix
Provide random access to cached NARs
E.g. $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m4.139s $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m0.024s (Before, the second call took ~0.220s.) This will use a NAR listing in /tmp/nars/b0w2hafndl09h64fhb86kw6bmhbmnpm1.ls containing all metadata, including the offsets of regular files inside the NAR. Thus, we don't need to read the entire NAR. (We do read the entire listing, but that's generally pretty small. We could use a SQLite DB by borrowing some more code from nixos-channel-scripts/file-cache.hh.) This is primarily useful when Hydra is serving files from an S3 binary cache, in particular when you have giant NARs. E.g. we had some 12 GiB NARs, so accessing individuals files was pretty slow.
This commit is contained in:
parent
338f29dbd4
commit
2df9cbeb47
|
@ -129,10 +129,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
|
|
||||||
auto narAccessor = makeNarAccessor(nar);
|
auto narAccessor = makeNarAccessor(nar);
|
||||||
|
|
||||||
if (accessor_) {
|
if (accessor_)
|
||||||
accessor_->nars.emplace(info.path, narAccessor);
|
accessor_->addToCache(info.path, *nar, narAccessor);
|
||||||
accessor_->addToCache(info.path, *nar);
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
{
|
||||||
auto res = jsonRoot.placeholder("root");
|
auto res = jsonRoot.placeholder("root");
|
||||||
|
@ -144,10 +142,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
}
|
}
|
||||||
|
|
||||||
else {
|
else {
|
||||||
if (accessor_) {
|
if (accessor_)
|
||||||
accessor_->nars.emplace(info.path, makeNarAccessor(nar));
|
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
|
||||||
accessor_->addToCache(info.path, *nar);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Compress the NAR. */
|
/* Compress the NAR. */
|
||||||
|
|
|
@ -6,6 +6,8 @@
|
||||||
#include <stack>
|
#include <stack>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
|
||||||
|
#include <nlohmann/json.hpp>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct NarMember
|
struct NarMember
|
||||||
|
@ -24,31 +26,37 @@ struct NarMember
|
||||||
std::map<std::string, NarMember> children;
|
std::map<std::string, NarMember> children;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct NarIndexer : ParseSink, StringSource
|
struct NarAccessor : public FSAccessor
|
||||||
{
|
{
|
||||||
|
std::shared_ptr<const std::string> nar;
|
||||||
|
|
||||||
|
GetNarBytes getNarBytes;
|
||||||
|
|
||||||
NarMember root;
|
NarMember root;
|
||||||
std::stack<NarMember*> parents;
|
|
||||||
|
struct NarIndexer : ParseSink, StringSource
|
||||||
|
{
|
||||||
|
NarAccessor & acc;
|
||||||
|
|
||||||
|
std::stack<NarMember *> parents;
|
||||||
|
|
||||||
std::string currentStart;
|
std::string currentStart;
|
||||||
bool isExec = false;
|
bool isExec = false;
|
||||||
|
|
||||||
NarIndexer(const std::string & nar) : StringSource(nar)
|
NarIndexer(NarAccessor & acc, const std::string & nar)
|
||||||
{
|
: StringSource(nar), acc(acc)
|
||||||
}
|
{ }
|
||||||
|
|
||||||
void createMember(const Path & path, NarMember member) {
|
void createMember(const Path & path, NarMember member) {
|
||||||
size_t level = std::count(path.begin(), path.end(), '/');
|
size_t level = std::count(path.begin(), path.end(), '/');
|
||||||
while(parents.size() > level) {
|
while (parents.size() > level) parents.pop();
|
||||||
parents.pop();
|
|
||||||
}
|
|
||||||
|
|
||||||
if(parents.empty()) {
|
if (parents.empty()) {
|
||||||
root = std::move(member);
|
acc.root = std::move(member);
|
||||||
parents.push(&root);
|
parents.push(&acc.root);
|
||||||
} else {
|
} else {
|
||||||
if(parents.top()->type != FSAccessor::Type::tDirectory) {
|
if (parents.top()->type != FSAccessor::Type::tDirectory)
|
||||||
throw Error(format("NAR file missing parent directory of path '%1%'") % path);
|
throw Error("NAR file missing parent directory of path '%s'", path);
|
||||||
}
|
|
||||||
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
|
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
|
||||||
parents.push(&result.first->second);
|
parents.push(&result.first->second);
|
||||||
}
|
}
|
||||||
|
@ -56,12 +64,12 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
|
|
||||||
void createDirectory(const Path & path) override
|
void createDirectory(const Path & path) override
|
||||||
{
|
{
|
||||||
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 });
|
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
|
||||||
}
|
}
|
||||||
|
|
||||||
void createRegularFile(const Path & path) override
|
void createRegularFile(const Path & path) override
|
||||||
{
|
{
|
||||||
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 });
|
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
|
||||||
}
|
}
|
||||||
|
|
||||||
void isExecutable() override
|
void isExecutable() override
|
||||||
|
@ -91,16 +99,54 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
createMember(path,
|
createMember(path,
|
||||||
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
NarMember* find(const Path & path)
|
NarAccessor(ref<const std::string> nar) : nar(nar)
|
||||||
|
{
|
||||||
|
NarIndexer indexer(*this, *nar);
|
||||||
|
parseDump(indexer, indexer);
|
||||||
|
}
|
||||||
|
|
||||||
|
NarAccessor(const std::string & listing, GetNarBytes getNarBytes)
|
||||||
|
: getNarBytes(getNarBytes)
|
||||||
|
{
|
||||||
|
using json = nlohmann::json;
|
||||||
|
|
||||||
|
std::function<void(NarMember &, json &)> recurse;
|
||||||
|
|
||||||
|
recurse = [&](NarMember & member, json & v) {
|
||||||
|
std::string type = v["type"];
|
||||||
|
|
||||||
|
if (type == "directory") {
|
||||||
|
member.type = FSAccessor::Type::tDirectory;
|
||||||
|
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
|
||||||
|
std::string name = i.key();
|
||||||
|
recurse(member.children[name], i.value());
|
||||||
|
}
|
||||||
|
} else if (type == "regular") {
|
||||||
|
member.type = FSAccessor::Type::tRegular;
|
||||||
|
member.size = v["size"];
|
||||||
|
member.isExecutable = v.value("executable", false);
|
||||||
|
member.start = v["narOffset"];
|
||||||
|
} else if (type == "symlink") {
|
||||||
|
member.type = FSAccessor::Type::tSymlink;
|
||||||
|
member.target = v.value("target", "");
|
||||||
|
} else return;
|
||||||
|
};
|
||||||
|
|
||||||
|
json v = json::parse(listing);
|
||||||
|
recurse(root, v);
|
||||||
|
}
|
||||||
|
|
||||||
|
NarMember * find(const Path & path)
|
||||||
{
|
{
|
||||||
Path canon = path == "" ? "" : canonPath(path);
|
Path canon = path == "" ? "" : canonPath(path);
|
||||||
NarMember* current = &root;
|
NarMember * current = &root;
|
||||||
auto end = path.end();
|
auto end = path.end();
|
||||||
for(auto it = path.begin(); it != end; ) {
|
for (auto it = path.begin(); it != end; ) {
|
||||||
// because it != end, the remaining component is non-empty so we need
|
// because it != end, the remaining component is non-empty so we need
|
||||||
// a directory
|
// a directory
|
||||||
if(current->type != FSAccessor::Type::tDirectory) return nullptr;
|
if (current->type != FSAccessor::Type::tDirectory) return nullptr;
|
||||||
|
|
||||||
// skip slash (canonPath above ensures that this is always a slash)
|
// skip slash (canonPath above ensures that this is always a slash)
|
||||||
assert(*it == '/');
|
assert(*it == '/');
|
||||||
|
@ -109,7 +155,7 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
// lookup current component
|
// lookup current component
|
||||||
auto next = std::find(it, end, '/');
|
auto next = std::find(it, end, '/');
|
||||||
auto child = current->children.find(std::string(it, next));
|
auto child = current->children.find(std::string(it, next));
|
||||||
if(child == current->children.end()) return nullptr;
|
if (child == current->children.end()) return nullptr;
|
||||||
current = &child->second;
|
current = &child->second;
|
||||||
|
|
||||||
it = next;
|
it = next;
|
||||||
|
@ -118,28 +164,16 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
return current;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
||||||
NarMember& at(const Path & path) {
|
NarMember & get(const Path & path) {
|
||||||
auto result = find(path);
|
auto result = find(path);
|
||||||
if(result == nullptr) {
|
if (result == nullptr)
|
||||||
throw Error(format("NAR file does not contain path '%1%'") % path);
|
throw Error("NAR file does not contain path '%1%'", path);
|
||||||
}
|
|
||||||
return *result;
|
return *result;
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
struct NarAccessor : public FSAccessor
|
|
||||||
{
|
|
||||||
ref<const std::string> nar;
|
|
||||||
NarIndexer indexer;
|
|
||||||
|
|
||||||
NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
|
|
||||||
{
|
|
||||||
parseDump(indexer, indexer);
|
|
||||||
}
|
|
||||||
|
|
||||||
Stat stat(const Path & path) override
|
Stat stat(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.find(path);
|
auto i = find(path);
|
||||||
if (i == nullptr)
|
if (i == nullptr)
|
||||||
return {FSAccessor::Type::tMissing, 0, false};
|
return {FSAccessor::Type::tMissing, 0, false};
|
||||||
return {i->type, i->size, i->isExecutable, i->start};
|
return {i->type, i->size, i->isExecutable, i->start};
|
||||||
|
@ -147,30 +181,33 @@ struct NarAccessor : public FSAccessor
|
||||||
|
|
||||||
StringSet readDirectory(const Path & path) override
|
StringSet readDirectory(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.at(path);
|
auto i = get(path);
|
||||||
|
|
||||||
if (i.type != FSAccessor::Type::tDirectory)
|
if (i.type != FSAccessor::Type::tDirectory)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
|
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
|
||||||
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
for(auto&& child : i.children) {
|
for (auto & child : i.children)
|
||||||
res.insert(child.first);
|
res.insert(child.first);
|
||||||
|
|
||||||
}
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readFile(const Path & path) override
|
std::string readFile(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.at(path);
|
auto i = get(path);
|
||||||
if (i.type != FSAccessor::Type::tRegular)
|
if (i.type != FSAccessor::Type::tRegular)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
|
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
|
||||||
|
|
||||||
|
if (getNarBytes) return getNarBytes(i.start, i.size);
|
||||||
|
|
||||||
|
assert(nar);
|
||||||
return std::string(*nar, i.start, i.size);
|
return std::string(*nar, i.start, i.size);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readLink(const Path & path) override
|
std::string readLink(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.at(path);
|
auto i = get(path);
|
||||||
if (i.type != FSAccessor::Type::tSymlink)
|
if (i.type != FSAccessor::Type::tSymlink)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
|
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
|
||||||
return i.target;
|
return i.target;
|
||||||
|
@ -182,6 +219,12 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
|
||||||
return make_ref<NarAccessor>(nar);
|
return make_ref<NarAccessor>(nar);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
|
||||||
|
GetNarBytes getNarBytes)
|
||||||
|
{
|
||||||
|
return make_ref<NarAccessor>(listing, getNarBytes);
|
||||||
|
}
|
||||||
|
|
||||||
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
||||||
const Path & path, bool recurse)
|
const Path & path, bool recurse)
|
||||||
{
|
{
|
||||||
|
|
|
@ -8,6 +8,16 @@ namespace nix {
|
||||||
file. */
|
file. */
|
||||||
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
|
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
|
||||||
|
|
||||||
|
/* Create a NAR accessor from a NAR listing (in the format produced by
|
||||||
|
listNar()). The callback getNarBytes(offset, length) is used by the
|
||||||
|
readFile() method of the accessor to get the contents of files
|
||||||
|
inside the NAR. */
|
||||||
|
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
|
||||||
|
|
||||||
|
ref<FSAccessor> makeLazyNarAccessor(
|
||||||
|
const std::string & listing,
|
||||||
|
GetNarBytes getNarBytes);
|
||||||
|
|
||||||
class JSONPlaceholder;
|
class JSONPlaceholder;
|
||||||
|
|
||||||
/* Write a JSON representation of the contents of a NAR (except file
|
/* Write a JSON representation of the contents of a NAR (except file
|
||||||
|
|
|
@ -1,5 +1,10 @@
|
||||||
#include "remote-fs-accessor.hh"
|
#include "remote-fs-accessor.hh"
|
||||||
#include "nar-accessor.hh"
|
#include "nar-accessor.hh"
|
||||||
|
#include "json.hh"
|
||||||
|
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
@ -11,21 +16,31 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
|
||||||
createDirs(cacheDir);
|
createDirs(cacheDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
Path RemoteFSAccessor::makeCacheFile(const Path & storePath)
|
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
|
||||||
{
|
{
|
||||||
assert(cacheDir != "");
|
assert(cacheDir != "");
|
||||||
return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath));
|
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
|
||||||
}
|
}
|
||||||
|
|
||||||
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar)
|
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
|
||||||
|
ref<FSAccessor> narAccessor)
|
||||||
{
|
{
|
||||||
|
nars.emplace(storePath, narAccessor);
|
||||||
|
|
||||||
|
if (cacheDir != "") {
|
||||||
try {
|
try {
|
||||||
if (cacheDir == "") return;
|
std::ostringstream str;
|
||||||
|
JSONPlaceholder jsonRoot(str);
|
||||||
|
listNar(jsonRoot, narAccessor, "", true);
|
||||||
|
writeFile(makeCacheFile(storePath, "ls"), str.str());
|
||||||
|
|
||||||
/* FIXME: do this asynchronously. */
|
/* FIXME: do this asynchronously. */
|
||||||
writeFile(makeCacheFile(storePath), nar);
|
writeFile(makeCacheFile(storePath, "nar"), nar);
|
||||||
|
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
ignoreException();
|
ignoreException();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
|
@ -42,20 +57,49 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
if (i != nars.end()) return {i->second, restPath};
|
if (i != nars.end()) return {i->second, restPath};
|
||||||
|
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
|
std::string listing;
|
||||||
|
Path cacheFile;
|
||||||
|
|
||||||
|
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (cacheDir != "")
|
listing = nix::readFile(makeCacheFile(storePath, "ls"));
|
||||||
*sink.s = nix::readFile(makeCacheFile(storePath));
|
|
||||||
|
auto narAccessor = makeLazyNarAccessor(listing,
|
||||||
|
[cacheFile](uint64_t offset, uint64_t length) {
|
||||||
|
|
||||||
|
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
|
||||||
|
if (!fd)
|
||||||
|
throw SysError("opening NAR cache file '%s'", cacheFile);
|
||||||
|
|
||||||
|
if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset)
|
||||||
|
throw SysError("seeking in '%s'", cacheFile);
|
||||||
|
|
||||||
|
std::string buf(length, 0);
|
||||||
|
readFull(fd.get(), (unsigned char *) buf.data(), length);
|
||||||
|
|
||||||
|
return buf;
|
||||||
|
});
|
||||||
|
|
||||||
|
nars.emplace(storePath, narAccessor);
|
||||||
|
return {narAccessor, restPath};
|
||||||
|
|
||||||
} catch (SysError &) { }
|
} catch (SysError &) { }
|
||||||
|
|
||||||
if (sink.s->empty()) {
|
try {
|
||||||
store->narFromPath(storePath, sink);
|
*sink.s = nix::readFile(cacheFile);
|
||||||
addToCache(storePath, *sink.s);
|
|
||||||
|
auto narAccessor = makeNarAccessor(sink.s);
|
||||||
|
nars.emplace(storePath, narAccessor);
|
||||||
|
return {narAccessor, restPath};
|
||||||
|
|
||||||
|
} catch (SysError &) { }
|
||||||
}
|
}
|
||||||
|
|
||||||
auto accessor = makeNarAccessor(sink.s);
|
store->narFromPath(storePath, sink);
|
||||||
nars.emplace(storePath, accessor);
|
auto narAccessor = makeNarAccessor(sink.s);
|
||||||
return {accessor, restPath};
|
addToCache(storePath, *sink.s, narAccessor);
|
||||||
|
return {narAccessor, restPath};
|
||||||
}
|
}
|
||||||
|
|
||||||
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
|
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
|
||||||
|
|
|
@ -18,9 +18,10 @@ class RemoteFSAccessor : public FSAccessor
|
||||||
|
|
||||||
friend class BinaryCacheStore;
|
friend class BinaryCacheStore;
|
||||||
|
|
||||||
Path makeCacheFile(const Path & storePath);
|
Path makeCacheFile(const Path & storePath, const std::string & ext);
|
||||||
|
|
||||||
void addToCache(const Path & storePath, const std::string & nar);
|
void addToCache(const Path & storePath, const std::string & nar,
|
||||||
|
ref<FSAccessor> narAccessor);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue