lix-project/lix
20
58
Fork 31
Code Issues 359 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Remove addPathToAccessor

Browse source
This commit is contained in:
Eelco Dolstra 2016-10-21 18:09:30 +02:00
parent 542ae5c8f8
commit fdbbcc4492
9 changed files with 97 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
src/libstore/binary-cache-store.cc
View file

@ -17,6 +17,66 @@
namespace nix {
/* Given requests for a path /nix/store/<x>/<y>, this accessor will
first download the NAR for /nix/store/<x> from the binary cache,
build a NAR accessor for that NAR, and use that to access <y>. */
struct BinaryCacheStoreAccessor : public FSAccessor
{
ref<BinaryCacheStore> store;
std::map<Path, ref<FSAccessor>> nars;
BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
: store(store)
{
}
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
store->narFromPath(storePath, sink);
auto accessor = makeNarAccessor(sink.s);
nars.emplace(storePath, accessor);
return {accessor, restPath};
}
Stat stat(const Path & path) override
{
auto res = fetch(path);
return res.first->stat(res.second);
}
StringSet readDirectory(const Path & path) override
{
auto res = fetch(path);
return res.first->readDirectory(res.second);
}
std::string readFile(const Path & path) override
{
auto res = fetch(path);
return res.first->readFile(res.second);
}
std::string readLink(const Path & path) override
{
auto res = fetch(path);
return res.first->readLink(res.second);
}
};
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
, compression(get(params, "compression", "xz"))
@ -82,7 +142,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
{
if (!repair && isValidPath(info.path)) return;
@ -109,6 +169,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
@ -118,12 +180,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1);
auto accessor = makeNarAccessor(nar);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->nars.emplace(info.path, narAccessor);
std::function<void(const Path &, JSONPlaceholder &)> recurse;
recurse = [&](const Path & path, JSONPlaceholder & res) {
auto st = accessor->stat(path);
auto st = narAccessor->stat(path);
auto obj = res.object();
@ -138,7 +203,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
obj.attr("type", "directory");
{
auto res2 = obj.object("entries");
for (auto & name : accessor->readDirectory(path)) {
for (auto & name : narAccessor->readDirectory(path)) {
auto res3 = res2.placeholder(name);
recurse(path + "/" + name, res3);
}
@ -146,7 +211,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
break;
case FSAccessor::Type::tSymlink:
obj.attr("type", "symlink");
obj.attr("target", accessor->readLink(path));
obj.attr("target", narAccessor->readLink(path));
break;
default:
abort();
@ -162,6 +227,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
}
else {
if (accessor_)
accessor_->nars.emplace(info.path, makeNarAccessor(nar));
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
@ -286,7 +356,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
addToStore(info, sink.s, repair);
addToStore(info, sink.s, repair, false, 0);
return info.path;
}
@ -301,84 +371,16 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(info.path)) {
StringSink sink;
dumpString(s, sink);
addToStore(info, sink.s, repair);
addToStore(info, sink.s, repair, false, 0);
}
return info.path;
}
/* Given requests for a path /nix/store/<x>/<y>, this accessor will
first download the NAR for /nix/store/<x> from the binary cache,
build a NAR accessor for that NAR, and use that to access <y>. */
struct BinaryCacheStoreAccessor : public FSAccessor
{
ref<BinaryCacheStore> store;
std::map<Path, ref<FSAccessor>> nars;
BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
: store(store)
{
}
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
store->narFromPath(storePath, sink);
auto accessor = makeNarAccessor(sink.s);
nars.emplace(storePath, accessor);
return {accessor, restPath};
}
Stat stat(const Path & path) override
{
auto res = fetch(path);
return res.first->stat(res.second);
}
StringSet readDirectory(const Path & path) override
{
auto res = fetch(path);
return res.first->readDirectory(res.second);
}
std::string readFile(const Path & path) override
{
auto res = fetch(path);
return res.first->readFile(res.second);
}
std::string readLink(const Path & path) override
{
auto res = fetch(path);
return res.first->readLink(res.second);
}
};
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{
return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>(
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
}
void BinaryCacheStore::addPathToAccessor(ref<FSAccessor> accessor,
const Path & storePath, const ref<std::string> & data)
{
auto accessor_ = accessor.dynamic_pointer_cast<BinaryCacheStoreAccessor>();
if (accessor_)
accessor_->nars.emplace(storePath, makeNarAccessor(data));
}
}

17
src/libstore/binary-cache-store.hh
View file

@ -93,22 +93,23 @@ public:
bool wantMassQuery() override { return wantMassQuery_; }
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair = false, bool dontCheckSigs = false) override;
bool repair, bool dontCheckSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, bool repair = false) override;
bool recursive, HashType hashAlgo,
PathFilter & filter, bool repair) override;
Path addTextToStore(const string & name, const string & s,
const PathSet & references, bool repair = false) override;
const PathSet & references, bool repair) override;
void narFromPath(const Path & path, Sink & sink) override;
void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override
void buildPaths(const PathSet & paths, BuildMode buildMode) override
{ notImpl(); }
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal) override
BuildMode buildMode) override
{ notImpl(); }
void ensurePath(const Path & path) override
@ -137,10 +138,6 @@ public:
ref<FSAccessor> getFSAccessor() override;
private:
void addPathToAccessor(ref<FSAccessor>, const Path & storePath, const ref<std::string> & data) override;
public:
void addSignatures(const Path & storePath, const StringSet & sigs) override

5
src/libstore/export-import.cc
View file

@ -117,10 +117,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
if (readInt(source) == 1)
readString(source);
addToStore(info, tee.data, false, dontCheckSigs);
if (accessor)
addPathToAccessor(ref<FSAccessor>(accessor), info.path, tee.data);
addToStore(info, tee.data, false, dontCheckSigs, accessor);
res.push_back(info.path);
}

2
src/libstore/local-store.cc
View file

@ -910,7 +910,7 @@ void LocalStore::invalidatePath(State & state, const Path & path)
void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
{
Hash h = hashString(htSHA256, *nar);
if (h != info.narHash)

9
src/libstore/local-store.hh
View file

@ -126,11 +126,12 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs) override;
bool repair, bool dontCheckSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, bool repair = false) override;
bool recursive, HashType hashAlgo,
PathFilter & filter, bool repair) override;
/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
@ -140,7 +141,7 @@ public:
bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false);
Path addTextToStore(const string & name, const string & s,
const PathSet & references, bool repair = false) override;
const PathSet & references, bool repair) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override;

2
src/libstore/remote-store.cc
View file

@ -333,7 +333,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
{
throw Error("RemoteStore::addToStore() not implemented");
}

3
src/libstore/remote-store.hh
View file

@ -54,7 +54,8 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs) override;
bool repair, bool dontCheckSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,

14
src/libstore/store-api.hh
View file

@ -366,7 +366,8 @@ public:
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair = false, bool dontCheckSigs = false) = 0;
bool repair = false, bool dontCheckSigs = false,
std::shared_ptr<FSAccessor> accessor = 0) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
@ -461,17 +462,6 @@ public:
/* Return an object to access files in the Nix store. */
virtual ref<FSAccessor> getFSAccessor() = 0;
private:
/* Inform an accessor about the NAR contents of a store path. Used
by importPaths() to speed up subsequent access to the imported
paths when used with binary cache stores. */
virtual void addPathToAccessor(ref<FSAccessor>, const Path & storePath, const ref<std::string> & data)
{
}
public:
/* Add signatures to the specified store path. The signatures are
not verified. */
virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0;

2
src/nix-daemon/nix-daemon.cc
View file

@ -304,7 +304,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
string s = readString(from);
PathSet refs = readStorePaths<PathSet>(*store, from);
startWork();
Path path = store->addTextToStore(suffix, s, refs);
Path path = store->addTextToStore(suffix, s, refs, false);
stopWork();
to << path;
break;