Use PathReferences more widely

This commit is contained in:
John Ericson 2020-10-07 13:52:20 +00:00
parent dae4409071
commit f8d562c0a7
29 changed files with 431 additions and 205 deletions

View file

@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32)
mXPUSHi(info->registrationTime); mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize); mXPUSHi(info->narSize);
AV * arr = newAV(); AV * arr = newAV();
for (auto & i : info->references) for (auto & i : info->referencesPossiblyToSelf())
av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) arr))); XPUSHs(sv_2mortal(newRV((SV *) arr)));
} catch (Error & e) { } catch (Error & e) {
@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
try { try {
auto h = Hash::parseAny(hash, parseHashType(algo)); auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name); auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht); Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
{
.method = ingestionMethod,
.hash = h,
},
{},
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.outputs.insert_or_assign("out", DerivationOutput {
.output = DerivationOutputCAFixed { .output = DerivationOutputCAFixed {
@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional<StorePath> expectedStorePath; std::optional<StorePath> expectedStorePath;
if (expectedHash) if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
Path dstPath; Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode dstPath = state.store->printStorePath(settings.readOnlyMode

View file

@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const
auto narHash = getNarHash(); auto narHash = getNarHash();
if (!narHash) if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string()); throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); return store.makeFixedOutputPath("source", FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
{},
});
} }
std::string Input::getType() const std::string Input::getType() const

View file

@ -71,14 +71,20 @@ DownloadFileResult downloadFile(
dumpString(*res.data, sink); dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data); auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), *store,
{
.name = name,
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Flat,
.hash = hash,
},
{},
},
},
hashString(htSHA256, *sink.s), hashString(htSHA256, *sink.s),
}; };
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = hash,
};
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);

View file

@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
unsupported("addToStoreFromDump"); unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name), *this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = nar.first,
},
{},
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
}); });
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, h, name), *this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info; return info;
})->path; })->path;
} }
@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto textHash = hashString(htSHA256, s); auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references); auto path = makeTextPath(name, TextInfo { textHash, references });
if (!repair && isValidPath(path)) if (!repair && isValidPath(path))
return path; return path;
auto source = StringSource { s }; auto source = StringSource { s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first }; ValidPathInfo info {
*this,
{
.name = name,
.info = TextInfo {
{ .hash = textHash },
references,
},
},
nar.first,
};
info.narSize = nar.second; info.narSize = nar.second;
info.ca = TextHash { textHash }; info.ca = TextHash { textHash };
info.references = references;
return info; return info;
})->path; })->path;
} }

View file

@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs()
break; break;
} }
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto refs = rewriteRefs();
HashModuloSink narSink { htSHA256, oldHashPart }; HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink); dumpPath(actualPath, narSink);
auto narHashAndSize = narSink.finish(); auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store.makeFixedOutputPath( worker.store,
outputHash.method, {
got, .name = outputPathName(drv->name, outputName),
outputPathName(drv->name, outputName), .info = FixedOutputInfo {
refs.references, {
refs.hasSelfReference), .method = outputHash.method,
.hash = got,
},
rewriteRefs(),
},
},
narHashAndSize.first, narHashAndSize.first,
}; };
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
static_cast<PathReferences<StorePath> &>(newInfo0) = refs;
assert(newInfo0.ca); assert(newInfo0.ca);
return newInfo0; return newInfo0;
@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext()
subs.pop_front(); subs.pop_front();
if (ca) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); subPath = sub->makeFixedOutputPathFromCA({
.name = std::string { storePath.name() },
.info = caWithoutRefs(*ca),
});
if (sub->storeDir == worker.store.storeDir) if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath); assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) { } else if (sub->storeDir != worker.store.storeDir) {
@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext()
} }
if (info->path != storePath) { if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) { if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) {
auto info2 = std::make_shared<ValidPathInfo>(*info); auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath; info2->path = storePath;
info = info2; info = info2;

View file

@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type); return makeFileIngestionPrefix(method) + printHashType(hash.type);
} }
std::string makeFileIngestionPrefix(const FileIngestionMethod m) std::string makeFileIngestionPrefix(const FileIngestionMethod m)
{ {
switch (m) { switch (m) {
@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
return ""; return "";
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
return "r:"; return "r:";
default:
throw Error("impossible, caught both cases");
} }
assert(false);
} }
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash th) { [](TextHash th) {
return "text:" + th.hash.to_string(Base32, true); return "text:"
+ th.hash.to_string(Base32, true);
}, },
[](FixedOutputHash fsh) { [](FixedOutputHash fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash); return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
} }
}, ca); }, ca);
} }
@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca)
}, },
[](FixedOutputHash fsh) { [](FixedOutputHash fsh) {
return fsh.hash; return fsh.hash;
} },
}, ca);
}
ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
return std::visit(overloaded {
[&](TextHash h) -> ContentAddressWithReferences {
return TextInfo { h, {}};
},
[&](FixedOutputHash h) -> ContentAddressWithReferences {
return FixedOutputInfo { h, {}};
},
}, ca); }, ca);
} }

View file

@ -2,14 +2,20 @@
#include <variant> #include <variant>
#include "hash.hh" #include "hash.hh"
#include "path.hh"
namespace nix { namespace nix {
/*
* Mini content address
*/
enum struct FileIngestionMethod : uint8_t { enum struct FileIngestionMethod : uint8_t {
Flat = false, Flat = false,
Recursive = true Recursive = true
}; };
struct TextHash { struct TextHash {
Hash hash; Hash hash;
}; };
@ -41,10 +47,6 @@ typedef std::variant<
ingested. */ ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m); std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(ContentAddress ca);
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
std::string renderContentAddressMethod(ContentAddressMethod caMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod);
/*
* References set
*/
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
bool operator == (const PathReferences<Ref> & other) const
{
return references == other.references
&& hasSelfReference == other.hasSelfReference;
}
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
/*
* Full content address
*
* See the schema for store paths in store-api.cc
*/
// This matches the additional info that we need for makeTextPath
struct TextInfo : TextHash {
// References for the paths, self references disallowed
StorePathSet references;
};
struct FixedOutputInfo : FixedOutputHash {
// References for the paths
PathReferences<StorePath> references;
};
typedef std::variant<
TextInfo,
FixedOutputInfo
> ContentAddressWithReferences;
ContentAddressWithReferences caWithoutRefs(const ContentAddress &);
struct StorePathDescriptor {
std::string name;
ContentAddressWithReferences info;
bool operator == (const StorePathDescriptor & other) const
{
return name == other.name;
// FIXME second field
}
bool operator < (const StorePathDescriptor & other) const
{
return name < other.name;
// FIXME second field
}
};
} }

View file

@ -27,8 +27,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath( return store.makeFixedOutputPath(
hash.method, hash.hash, outputPathName(drvName, outputName),
outputPathName(drvName, outputName)); { hash, {} });
} }

View file

@ -130,8 +130,8 @@ struct Derivation : BasicDerivation
/* Return the underlying basic derivation but with these changes: /* Return the underlying basic derivation but with these changes:
1. Input drvs are emptied, but the outputs of them that were used are 1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources. added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */ 2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store); std::optional<BasicDerivation> tryResolve(Store & store);

View file

@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} });
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](DerivationOutputCAFloating _) { [&](DerivationOutputCAFloating _) {
@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
// recompute store path so that we can use a different store root // recompute store path so that we can use a different store root
if (path.second) { if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); subPath = makeFixedOutputPathFromCA({
.name = std::string { path.first.name() },
.info = caWithoutRefs(*path.second),
});
if (sub->storeDir == storeDir) if (sub->storeDir == storeDir)
assert(subPath == path.first); assert(subPath == path.first);
if (subPath != path.first) if (subPath != path.first)
@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto [hash, size] = hashSink->finish(); auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name); auto desc = StorePathDescriptor {
name,
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
};
auto dstPath = makeFixedOutputPathFromCA(desc);
addTempRoot(dstPath); addTempRoot(dstPath);
@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
autoGC(); autoGC();
if (inMemory) { if (inMemory) {
StringSource dumpSource { dump }; StringSource dumpSource { dump };
/* Restore from the NAR in memory. */ /* Restore from the NAR in memory. */
if (method == FileIngestionMethod::Recursive) if (method == FileIngestionMethod::Recursive)
restorePath(realPath, dumpSource); restorePath(realPath, dumpSource);
@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
optimisePath(realPath); optimisePath(realPath);
ValidPathInfo info { dstPath, narHash.first }; ValidPathInfo info { *this, std::move(desc), narHash.first };
info.narSize = narHash.second; info.narSize = narHash.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info); registerValidPath(info);
} }
@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto hash = hashString(htSHA256, s); auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references); auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
});
addTempRoot(dstPath); addTempRoot(dstPath);

View file

@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
std::string system; std::string system;
NarInfo() = delete; NarInfo() = delete;
NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash)
: ValidPathInfo(store, std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence); NarInfo(const Store & store, const std::string & s, const std::string & whence);

View file

@ -13,47 +13,6 @@ namespace nix {
class Store; class Store;
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
struct SubstitutablePathInfo : PathReferences<StorePath> struct SubstitutablePathInfo : PathReferences<StorePath>
{ {
@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences<StorePath>
{ {
StorePath path; StorePath path;
std::optional<StorePath> deriver; std::optional<StorePath> deriver;
// TODO document this
Hash narHash; Hash narHash;
time_t registrationTime = 0; time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown uint64_t narSize = 0; // 0 = unknown
@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences<StorePath>
void sign(const Store & store, const SecretKey & secretKey); void sign(const Store & store, const SecretKey & secretKey);
std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const;
/* Return true iff the path is verifiably content-addressed. */ /* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const; bool isContentAddressed(const Store & store) const;
@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences<StorePath>
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(const Store & store,
StorePathDescriptor && ca, Hash narHash);
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
}; };

View file

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "content-address.hh" #include <string_view>
#include "types.hh" #include "types.hh"
namespace nix { namespace nix {
@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths; typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap; typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";

View file

@ -7,6 +7,7 @@
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh" #include "json.hh"
#include "url.hh" #include "url.hh"
#include "references.hh"
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id,
} }
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
static std::string makeType( static std::string makeType(
const Store & store, const Store & store,
string && type, string && type,
const StorePathSet & references, const PathReferences<StorePath> & references)
bool hasSelfReference = false)
{ {
for (auto & i : references) { for (auto & i : references.references) {
type += ":"; type += ":";
type += store.printStorePath(i); type += store.printStorePath(i);
} }
if (hasSelfReference) type += ":self"; if (references.hasSelfReference) type += ":self";
return std::move(type); return std::move(type);
} }
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
} else { } else {
assert(references.empty()); assert(info.references.references.size() == 0);
assert(!info.references.hasSelfReference);
return makeStorePath("output:out", return makeStorePath("output:out",
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ makeFileIngestionPrefix(method) + makeFileIngestionPrefix(info.method)
+ hash.to_string(Base16, true) + ":"), + info.hash.to_string(Base16, true) + ":"),
name); name);
} }
} }
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", PathReferences<StorePath> { info.references }),
info.hash,
name);
}
StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const
{ {
// New template // New template
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHash th) { [&](TextInfo ti) {
return makeTextPath(name, th.hash, references); return makeTextPath(desc.name, ti);
}, },
[&](FixedOutputHash fsh) { [&](FixedOutputInfo foi) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); return makeFixedOutputPath(desc.name, foi);
} }
}, ca); }, desc.info);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType(*this, "text", references), hash, name);
} }
@ -229,14 +228,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
Hash h = method == FileIngestionMethod::Recursive Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath); : hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h); FixedOutputInfo caInfo {
{
.method = method,
.hash = h,
},
{},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
} }
StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const const StorePathSet & references) const
{ {
return makeTextPath(name, hashString(htSHA256, s), references); return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
});
} }
@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, hash, name), *this,
StorePathDescriptor {
std::string { name },
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
},
narHash, narHash,
}; };
info.narSize = narSize; info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = sinkToSource([&](Sink & scratchpadSink) {
@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath, queryPathInfoUncached(storePath,
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
} }
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) { if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++; stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePathS); throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
} }
(*callbackPtr)(ref<const ValidPathInfo>(info)); (*callbackPtr)(ref<const ValidPathInfo>(info));
@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
std::condition_variable wakeup; std::condition_variable wakeup;
ThreadPool pool; ThreadPool pool;
auto doQuery = [&](const Path & path) { auto doQuery = [&](const StorePath & path) {
checkInterrupt(); checkInterrupt();
queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
auto state(state_.lock()); auto state(state_.lock());
try { try {
auto info = fut.get(); auto info = fut.get();
state->valid.insert(parseStorePath(path)); state->valid.insert(path);
} catch (InvalidPath &) { } catch (InvalidPath &) {
} catch (...) { } catch (...) {
state->exc = std::current_exception(); state->exc = std::current_exception();
@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
}; };
for (auto & path : paths) for (auto & path : paths)
pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME pool.enqueue(std::bind(doQuery, path));
pool.process(); pool.process();
@ -737,7 +753,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
// recompute store path on the chance dstStore does it differently // recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info); auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); info2->path = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path); assert(info->path == info2->path);
info = info2; info = info2;
@ -799,7 +816,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
@ -826,7 +844,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir) if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store))); sigs.insert(secretKey.signDetached(fingerprint(store)));
} }
std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const
{
if (! ca)
return std::nullopt;
return StorePathDescriptor {
.name = std::string { path.name() },
.info = std::visit(overloaded {
[&](TextHash th) {
TextInfo info { th };
assert(!hasSelfReference);
info.references = references;
return ContentAddressWithReferences { info };
},
[&](FixedOutputHash foh) {
FixedOutputInfo info { foh };
info.references = static_cast<PathReferences<StorePath>>(*this);
return ContentAddressWithReferences { info };
},
}, *ca),
};
}
bool ValidPathInfo::isContentAddressed(const Store & store) const bool ValidPathInfo::isContentAddressed(const Store & store) const
{ {
if (! ca) return false; auto fullCaOpt = fullStorePathDescriptorOpt();
auto caPath = std::visit(overloaded { if (! fullCaOpt)
[&](TextHash th) { return false;
assert(!hasSelfReference);
return store.makeTextPath(path.name(), th.hash, references); auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt);
},
[&](FixedOutputHash fsh) {
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference);
}
}, *ca);
bool res = caPath == path; bool res = caPath == path;
@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const
} }
ValidPathInfo::ValidPathInfo(
const Store & store,
StorePathDescriptor && info,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(info))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo ti) {
this->references = ti.references;
this->ca = TextHash { std::move(ti) };
},
[this](FixedOutputInfo foi) {
*(static_cast<PathReferences<StorePath> *>(this)) = foi.references;
this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) };
},
}, std::move(info.info));
}
Derivation Store::derivationFromPath(const StorePath & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath)
{ {
ensurePath(drvPath); ensurePath(drvPath);

View file

@ -170,6 +170,8 @@ struct BuildResult
} }
}; };
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public Config struct StoreConfig : public Config
{ {
using Config::Config; using Config::Config;
@ -313,17 +315,11 @@ public:
StorePath makeOutputPath(std::string_view id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash, StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const;
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the /* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store

View file

@ -201,9 +201,8 @@ public:
template<typename... Args> template<typename... Args>
SysError(const Args & ... args) SysError(const Args & ... args)
: Error("") : Error(""), errNo(errno)
{ {
errNo = errno;
auto hf = hintfmt(args...); auto hf = hintfmt(args...);
err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
} }

View file

@ -4,6 +4,7 @@
#include <list> #include <list>
#include <set> #include <set>
#include <string>
#include <map> #include <map>
#include <vector> #include <vector>

View file

@ -161,8 +161,14 @@ static int _main(int argc, char * * argv)
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash::parseAny(args[1], ht); expectedHash = Hash::parseAny(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); storePath = store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = *expectedHash; hash = *expectedHash;
else else

View file

@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */ /* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs) static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{ {
auto recursive = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
for (auto i : opFlags) for (auto i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive; if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i); else throw UsageError("unknown flag '%1%'", i);
if (opArgs.size() != 3) if (opArgs.size() != 3)
@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
string hash = *i++; string hash = *i++;
string name = *i++; string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
{},
})));
} }

View file

@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand
} }
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(ingestionMethod, hash, *namePart), *store,
StorePathDescriptor {
.name = *namePart,
.info = FixedOutputInfo {
{
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
{},
},
},
narHash, narHash,
}; };
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) { if (!dryRun) {
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };

View file

@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
StringMap rewrites; StringMap rewrites;
StorePathSet references; PathReferences<StorePath> refs;
bool hasSelfReference = false; refs.hasSelfReference = oldInfo->hasSelfReference;
for (auto & ref : oldInfo->references) { for (auto & ref : oldInfo->references) {
if (ref == path) auto i = remappings.find(ref);
hasSelfReference = true; auto replacement = i != remappings.end() ? i->second : ref;
else { // FIXME: warn about unremapped paths?
auto i = remappings.find(ref); if (replacement != ref)
auto replacement = i != remappings.end() ? i->second : ref; rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
// FIXME: warn about unremapped paths? refs.references.insert(std::move(replacement));
if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
references.insert(std::move(replacement));
}
} }
*sink.s = rewriteStrings(*sink.s, rewrites); *sink.s = rewriteStrings(*sink.s, rewrites);
@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
auto narHash = hashModuloSink.finish().first; auto narHash = hashModuloSink.finish().first;
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), *store,
StorePathDescriptor {
.name = std::string { path.name() },
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
std::move(refs),
},
},
narHash, narHash,
}; };
info.references = std::move(references);
info.hasSelfReference = std::move(hasSelfReference);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
if (!json) if (!json)
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));

View file

@ -130,12 +130,21 @@ struct ProfileManifest
auto narHash = hashString(htSHA256, *sink.s); auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), *store,
StorePathDescriptor {
"profile",
FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
{ references },
},
},
narHash, narHash,
}; };
info.references = std::move(references); info.references = std::move(references);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
store->addToStore(info, source); store->addToStore(info, source);

View file

@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand
ThreadPool pool; ThreadPool pool;
auto doPath = [&](const Path & storePath) { auto doPath = [&](const StorePath & storePath) {
try { try {
checkInterrupt(); checkInterrupt();
MaintainCount<std::atomic<size_t>> mcActive(active); MaintainCount<std::atomic<size_t>> mcActive(active);
update(); update();
auto info = store->queryPathInfo(store->parseStorePath(storePath)); auto info = store->queryPathInfo(storePath);
// Note: info->path can be different from storePath // Note: info->path can be different from storePath
// for binary cache stores when using --all (since we // for binary cache stores when using --all (since we
@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand
}; };
for (auto & storePath : storePaths) for (auto & storePath : storePaths)
pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.enqueue(std::bind(doPath, storePath));
pool.process(); pool.process();