Use PathReferences more widely

This commit is contained in:
John Ericson 2020-10-07 13:52:20 +00:00
parent dae4409071
commit f8d562c0a7
29 changed files with 431 additions and 205 deletions

View file

@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32)
mXPUSHi(info->registrationTime); mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize); mXPUSHi(info->narSize);
AV * arr = newAV(); AV * arr = newAV();
for (auto & i : info->references) for (auto & i : info->referencesPossiblyToSelf())
av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) arr))); XPUSHs(sv_2mortal(newRV((SV *) arr)));
} catch (Error & e) { } catch (Error & e) {
@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
try { try {
auto h = Hash::parseAny(hash, parseHashType(algo)); auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name); auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht); Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
{
.method = ingestionMethod,
.hash = h,
},
{},
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.outputs.insert_or_assign("out", DerivationOutput {
.output = DerivationOutputCAFixed { .output = DerivationOutputCAFixed {
@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional<StorePath> expectedStorePath; std::optional<StorePath> expectedStorePath;
if (expectedHash) if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
Path dstPath; Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode dstPath = state.store->printStorePath(settings.readOnlyMode

View file

@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const
auto narHash = getNarHash(); auto narHash = getNarHash();
if (!narHash) if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string()); throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); return store.makeFixedOutputPath("source", FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
{},
});
} }
std::string Input::getType() const std::string Input::getType() const

View file

@ -71,14 +71,20 @@ DownloadFileResult downloadFile(
dumpString(*res.data, sink); dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data); auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), *store,
{
.name = name,
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Flat,
.hash = hash,
},
{},
},
},
hashString(htSHA256, *sink.s), hashString(htSHA256, *sink.s),
}; };
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = hash,
};
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);

View file

@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
unsupported("addToStoreFromDump"); unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name), *this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = nar.first,
},
{},
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
}); });
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, h, name), *this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info; return info;
})->path; })->path;
} }
@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto textHash = hashString(htSHA256, s); auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references); auto path = makeTextPath(name, TextInfo { textHash, references });
if (!repair && isValidPath(path)) if (!repair && isValidPath(path))
return path; return path;
auto source = StringSource { s }; auto source = StringSource { s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first }; ValidPathInfo info {
*this,
{
.name = name,
.info = TextInfo {
{ .hash = textHash },
references,
},
},
nar.first,
};
info.narSize = nar.second; info.narSize = nar.second;
info.ca = TextHash { textHash }; info.ca = TextHash { textHash };
info.references = references;
return info; return info;
})->path; })->path;
} }

View file

@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs()
break; break;
} }
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto refs = rewriteRefs();
HashModuloSink narSink { htSHA256, oldHashPart }; HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink); dumpPath(actualPath, narSink);
auto narHashAndSize = narSink.finish(); auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store.makeFixedOutputPath( worker.store,
outputHash.method, {
got, .name = outputPathName(drv->name, outputName),
outputPathName(drv->name, outputName), .info = FixedOutputInfo {
refs.references, {
refs.hasSelfReference), .method = outputHash.method,
.hash = got,
},
rewriteRefs(),
},
},
narHashAndSize.first, narHashAndSize.first,
}; };
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
static_cast<PathReferences<StorePath> &>(newInfo0) = refs;
assert(newInfo0.ca); assert(newInfo0.ca);
return newInfo0; return newInfo0;
@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext()
subs.pop_front(); subs.pop_front();
if (ca) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); subPath = sub->makeFixedOutputPathFromCA({
.name = std::string { storePath.name() },
.info = caWithoutRefs(*ca),
});
if (sub->storeDir == worker.store.storeDir) if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath); assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) { } else if (sub->storeDir != worker.store.storeDir) {
@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext()
} }
if (info->path != storePath) { if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) { if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) {
auto info2 = std::make_shared<ValidPathInfo>(*info); auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath; info2->path = storePath;
info = info2; info = info2;

View file

@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type); return makeFileIngestionPrefix(method) + printHashType(hash.type);
} }
std::string makeFileIngestionPrefix(const FileIngestionMethod m) std::string makeFileIngestionPrefix(const FileIngestionMethod m)
{ {
switch (m) { switch (m) {
@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
return ""; return "";
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
return "r:"; return "r:";
default:
throw Error("impossible, caught both cases");
} }
assert(false);
} }
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash th) { [](TextHash th) {
return "text:" + th.hash.to_string(Base32, true); return "text:"
+ th.hash.to_string(Base32, true);
}, },
[](FixedOutputHash fsh) { [](FixedOutputHash fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash); return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
} }
}, ca); }, ca);
} }
@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca)
}, },
[](FixedOutputHash fsh) { [](FixedOutputHash fsh) {
return fsh.hash; return fsh.hash;
},
}, ca);
} }
ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
return std::visit(overloaded {
[&](TextHash h) -> ContentAddressWithReferences {
return TextInfo { h, {}};
},
[&](FixedOutputHash h) -> ContentAddressWithReferences {
return FixedOutputInfo { h, {}};
},
}, ca); }, ca);
} }

View file

@ -2,14 +2,20 @@
#include <variant> #include <variant>
#include "hash.hh" #include "hash.hh"
#include "path.hh"
namespace nix { namespace nix {
/*
* Mini content address
*/
enum struct FileIngestionMethod : uint8_t { enum struct FileIngestionMethod : uint8_t {
Flat = false, Flat = false,
Recursive = true Recursive = true
}; };
struct TextHash { struct TextHash {
Hash hash; Hash hash;
}; };
@ -41,10 +47,6 @@ typedef std::variant<
ingested. */ ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m); std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(ContentAddress ca);
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
std::string renderContentAddressMethod(ContentAddressMethod caMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod);
/*
* References set
*/
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
bool operator == (const PathReferences<Ref> & other) const
{
return references == other.references
&& hasSelfReference == other.hasSelfReference;
}
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
/*
* Full content address
*
* See the schema for store paths in store-api.cc
*/
// This matches the additional info that we need for makeTextPath
struct TextInfo : TextHash {
// References for the paths, self references disallowed
StorePathSet references;
};
struct FixedOutputInfo : FixedOutputHash {
// References for the paths
PathReferences<StorePath> references;
};
typedef std::variant<
TextInfo,
FixedOutputInfo
> ContentAddressWithReferences;
ContentAddressWithReferences caWithoutRefs(const ContentAddress &);
struct StorePathDescriptor {
std::string name;
ContentAddressWithReferences info;
bool operator == (const StorePathDescriptor & other) const
{
return name == other.name;
// FIXME second field
}
bool operator < (const StorePathDescriptor & other) const
{
return name < other.name;
// FIXME second field
}
};
} }

View file

@ -27,8 +27,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath( return store.makeFixedOutputPath(
hash.method, hash.hash, outputPathName(drvName, outputName),
outputPathName(drvName, outputName)); { hash, {} });
} }

View file

@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} });
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](DerivationOutputCAFloating _) { [&](DerivationOutputCAFloating _) {
@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
// recompute store path so that we can use a different store root // recompute store path so that we can use a different store root
if (path.second) { if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); subPath = makeFixedOutputPathFromCA({
.name = std::string { path.first.name() },
.info = caWithoutRefs(*path.second),
});
if (sub->storeDir == storeDir) if (sub->storeDir == storeDir)
assert(subPath == path.first); assert(subPath == path.first);
if (subPath != path.first) if (subPath != path.first)
@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto [hash, size] = hashSink->finish(); auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name); auto desc = StorePathDescriptor {
name,
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
};
auto dstPath = makeFixedOutputPathFromCA(desc);
addTempRoot(dstPath); addTempRoot(dstPath);
@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
optimisePath(realPath); optimisePath(realPath);
ValidPathInfo info { dstPath, narHash.first }; ValidPathInfo info { *this, std::move(desc), narHash.first };
info.narSize = narHash.second; info.narSize = narHash.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info); registerValidPath(info);
} }
@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto hash = hashString(htSHA256, s); auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references); auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
});
addTempRoot(dstPath); addTempRoot(dstPath);

View file

@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
std::string system; std::string system;
NarInfo() = delete; NarInfo() = delete;
NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash)
: ValidPathInfo(store, std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence); NarInfo(const Store & store, const std::string & s, const std::string & whence);

View file

@ -13,47 +13,6 @@ namespace nix {
class Store; class Store;
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
struct SubstitutablePathInfo : PathReferences<StorePath> struct SubstitutablePathInfo : PathReferences<StorePath>
{ {
@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences<StorePath>
{ {
StorePath path; StorePath path;
std::optional<StorePath> deriver; std::optional<StorePath> deriver;
// TODO document this
Hash narHash; Hash narHash;
time_t registrationTime = 0; time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown uint64_t narSize = 0; // 0 = unknown
@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences<StorePath>
void sign(const Store & store, const SecretKey & secretKey); void sign(const Store & store, const SecretKey & secretKey);
std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const;
/* Return true iff the path is verifiably content-addressed. */ /* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const; bool isContentAddressed(const Store & store) const;
@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences<StorePath>
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(const Store & store,
StorePathDescriptor && ca, Hash narHash);
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
}; };

View file

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "content-address.hh" #include <string_view>
#include "types.hh" #include "types.hh"
namespace nix { namespace nix {
@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths; typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap; typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";

View file

@ -7,6 +7,7 @@
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh" #include "json.hh"
#include "url.hh" #include "url.hh"
#include "references.hh"
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id,
} }
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
static std::string makeType( static std::string makeType(
const Store & store, const Store & store,
string && type, string && type,
const StorePathSet & references, const PathReferences<StorePath> & references)
bool hasSelfReference = false)
{ {
for (auto & i : references) { for (auto & i : references.references) {
type += ":"; type += ":";
type += store.printStorePath(i); type += store.printStorePath(i);
} }
if (hasSelfReference) type += ":self"; if (references.hasSelfReference) type += ":self";
return std::move(type); return std::move(type);
} }
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
} else { } else {
assert(references.empty()); assert(info.references.references.size() == 0);
assert(!info.references.hasSelfReference);
return makeStorePath("output:out", return makeStorePath("output:out",
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ makeFileIngestionPrefix(method) + makeFileIngestionPrefix(info.method)
+ hash.to_string(Base16, true) + ":"), + info.hash.to_string(Base16, true) + ":"),
name); name);
} }
} }
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", PathReferences<StorePath> { info.references }),
info.hash,
name);
}
StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const
{ {
// New template // New template
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHash th) { [&](TextInfo ti) {
return makeTextPath(name, th.hash, references); return makeTextPath(desc.name, ti);
}, },
[&](FixedOutputHash fsh) { [&](FixedOutputInfo foi) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); return makeFixedOutputPath(desc.name, foi);
} }
}, ca); }, desc.info);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType(*this, "text", references), hash, name);
} }
@ -229,14 +228,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
Hash h = method == FileIngestionMethod::Recursive Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath); : hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h); FixedOutputInfo caInfo {
{
.method = method,
.hash = h,
},
{},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
} }
StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const const StorePathSet & references) const
{ {
return makeTextPath(name, hashString(htSHA256, s), references); return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
});
} }
@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, hash, name), *this,
StorePathDescriptor {
std::string { name },
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
},
narHash, narHash,
}; };
info.narSize = narSize; info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = sinkToSource([&](Sink & scratchpadSink) {
@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath, queryPathInfoUncached(storePath,
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
} }
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) { if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++; stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePathS); throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
} }
(*callbackPtr)(ref<const ValidPathInfo>(info)); (*callbackPtr)(ref<const ValidPathInfo>(info));
@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
std::condition_variable wakeup; std::condition_variable wakeup;
ThreadPool pool; ThreadPool pool;
auto doQuery = [&](const Path & path) { auto doQuery = [&](const StorePath & path) {
checkInterrupt(); checkInterrupt();
queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
auto state(state_.lock()); auto state(state_.lock());
try { try {
auto info = fut.get(); auto info = fut.get();
state->valid.insert(parseStorePath(path)); state->valid.insert(path);
} catch (InvalidPath &) { } catch (InvalidPath &) {
} catch (...) { } catch (...) {
state->exc = std::current_exception(); state->exc = std::current_exception();
@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
}; };
for (auto & path : paths) for (auto & path : paths)
pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME pool.enqueue(std::bind(doQuery, path));
pool.process(); pool.process();
@ -737,7 +753,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
// recompute store path on the chance dstStore does it differently // recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info); auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); info2->path = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path); assert(info->path == info2->path);
info = info2; info = info2;
@ -799,7 +816,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
@ -826,7 +844,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store))); sigs.insert(secretKey.signDetached(fingerprint(store)));
} }
std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const
{
if (! ca)
return std::nullopt;
return StorePathDescriptor {
.name = std::string { path.name() },
.info = std::visit(overloaded {
[&](TextHash th) {
TextInfo info { th };
assert(!hasSelfReference);
info.references = references;
return ContentAddressWithReferences { info };
},
[&](FixedOutputHash foh) {
FixedOutputInfo info { foh };
info.references = static_cast<PathReferences<StorePath>>(*this);
return ContentAddressWithReferences { info };
},
}, *ca),
};
}
bool ValidPathInfo::isContentAddressed(const Store & store) const bool ValidPathInfo::isContentAddressed(const Store & store) const
{ {
if (! ca) return false; auto fullCaOpt = fullStorePathDescriptorOpt();
auto caPath = std::visit(overloaded { if (! fullCaOpt)
[&](TextHash th) { return false;
assert(!hasSelfReference);
return store.makeTextPath(path.name(), th.hash, references); auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt);
},
[&](FixedOutputHash fsh) {
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference);
}
}, *ca);
bool res = caPath == path; bool res = caPath == path;
@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const
} }
ValidPathInfo::ValidPathInfo(
const Store & store,
StorePathDescriptor && info,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(info))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo ti) {
this->references = ti.references;
this->ca = TextHash { std::move(ti) };
},
[this](FixedOutputInfo foi) {
*(static_cast<PathReferences<StorePath> *>(this)) = foi.references;
this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) };
},
}, std::move(info.info));
}
Derivation Store::derivationFromPath(const StorePath & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath)
{ {
ensurePath(drvPath); ensurePath(drvPath);

View file

@ -170,6 +170,8 @@ struct BuildResult
} }
}; };
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public Config struct StoreConfig : public Config
{ {
using Config::Config; using Config::Config;
@ -313,17 +315,11 @@ public:
StorePath makeOutputPath(std::string_view id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash, StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const;
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the /* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store

View file

@ -201,9 +201,8 @@ public:
template<typename... Args> template<typename... Args>
SysError(const Args & ... args) SysError(const Args & ... args)
: Error("") : Error(""), errNo(errno)
{ {
errNo = errno;
auto hf = hintfmt(args...); auto hf = hintfmt(args...);
err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
} }

View file

@ -4,6 +4,7 @@
#include <list> #include <list>
#include <set> #include <set>
#include <string>
#include <map> #include <map>
#include <vector> #include <vector>

View file

@ -161,8 +161,14 @@ static int _main(int argc, char * * argv)
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash::parseAny(args[1], ht); expectedHash = Hash::parseAny(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); storePath = store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = *expectedHash; hash = *expectedHash;
else else

View file

@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */ /* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs) static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{ {
auto recursive = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
for (auto i : opFlags) for (auto i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive; if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i); else throw UsageError("unknown flag '%1%'", i);
if (opArgs.size() != 3) if (opArgs.size() != 3)
@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
string hash = *i++; string hash = *i++;
string name = *i++; string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
{},
})));
} }

View file

@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand
} }
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(ingestionMethod, hash, *namePart), *store,
StorePathDescriptor {
.name = *namePart,
.info = FixedOutputInfo {
{
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
{},
},
},
narHash, narHash,
}; };
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) { if (!dryRun) {
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };

View file

@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
StringMap rewrites; StringMap rewrites;
StorePathSet references; PathReferences<StorePath> refs;
bool hasSelfReference = false; refs.hasSelfReference = oldInfo->hasSelfReference;
for (auto & ref : oldInfo->references) { for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref); auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref; auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths? // FIXME: warn about unremapped paths?
if (replacement != ref) if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
references.insert(std::move(replacement)); refs.references.insert(std::move(replacement));
}
} }
*sink.s = rewriteStrings(*sink.s, rewrites); *sink.s = rewriteStrings(*sink.s, rewrites);
@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
auto narHash = hashModuloSink.finish().first; auto narHash = hashModuloSink.finish().first;
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), *store,
StorePathDescriptor {
.name = std::string { path.name() },
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
std::move(refs),
},
},
narHash, narHash,
}; };
info.references = std::move(references);
info.hasSelfReference = std::move(hasSelfReference);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
if (!json) if (!json)
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));

View file

@ -130,12 +130,21 @@ struct ProfileManifest
auto narHash = hashString(htSHA256, *sink.s); auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), *store,
StorePathDescriptor {
"profile",
FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
{ references },
},
},
narHash, narHash,
}; };
info.references = std::move(references); info.references = std::move(references);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
store->addToStore(info, source); store->addToStore(info, source);

View file

@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand
ThreadPool pool; ThreadPool pool;
auto doPath = [&](const Path & storePath) { auto doPath = [&](const StorePath & storePath) {
try { try {
checkInterrupt(); checkInterrupt();
MaintainCount<std::atomic<size_t>> mcActive(active); MaintainCount<std::atomic<size_t>> mcActive(active);
update(); update();
auto info = store->queryPathInfo(store->parseStorePath(storePath)); auto info = store->queryPathInfo(storePath);
// Note: info->path can be different from storePath // Note: info->path can be different from storePath
// for binary cache stores when using --all (since we // for binary cache stores when using --all (since we
@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand
}; };
for (auto & storePath : storePaths) for (auto & storePath : storePaths)
pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.enqueue(std::bind(doPath, storePath));
pool.process(); pool.process();