Merge pull request #3746 from obsidiansystems/path-info

Introduce `StoreReferences` and `ContentAddressWithReferences`
This commit is contained in:
Robert Hensing 2023-04-17 15:49:48 +02:00 committed by GitHub
commit e641de085b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 578 additions and 243 deletions

View file

@ -293,7 +293,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
try { try {
auto h = Hash::parseAny(hash, parseHashType(algo)); auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name); auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -1293,7 +1293,13 @@ drvName, Bindings * attrs, Value & v)
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(method, h, drvName); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", drv.outputs.insert_or_assign("out",
DerivationOutput::CAFixed { DerivationOutput::CAFixed {
@ -2103,7 +2109,13 @@ static void addPath(
std::optional<StorePath> expectedStorePath; std::optional<StorePath> expectedStorePath;
if (expectedHash) if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = *expectedHash,
},
.references = {},
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode StorePath dstPath = settings.readOnlyMode

View file

@ -243,10 +243,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// early exit if pinned and already in the store // early exit if pinned and already in the store
if (expectedHash && expectedHash->type == htSHA256) { if (expectedHash && expectedHash->type == htSHA256) {
auto expectedPath = auto expectedPath = state.store->makeFixedOutputPath(
unpack name,
? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) FixedOutputInfo {
: state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); .hash = {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
},
.references = {}
});
if (state.store->isValidPath(expectedPath)) { if (state.store->isValidPath(expectedPath)) {
state.allowAndSetStorePathString(expectedPath, v); state.allowAndSetStorePathString(expectedPath, v);

View file

@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const
auto narHash = getNarHash(); auto narHash = getNarHash();
if (!narHash) if (!narHash)
throw Error("cannot compute store path for unlocked input '%s'", to_string()); throw Error("cannot compute store path for unlocked input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); return store.makeFixedOutputPath(getName(), FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
.references = {},
});
} }
std::string Input::getType() const std::string Input::getType() const

View file

@ -71,15 +71,19 @@ DownloadFileResult downloadFile(
dumpString(res.data, sink); dumpString(res.data, sink);
auto hash = hashString(htSHA256, res.data); auto hash = hashString(htSHA256, res.data);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), *store,
name,
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Flat,
.hash = hash,
},
.references = {},
},
hashString(htSHA256, sink.s), hashString(htSHA256, sink.s),
}; };
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash { auto source = StringSource { sink.s };
.method = FileIngestionMethod::Flat,
.hash = hash,
};
auto source = StringSource(sink.s);
store->addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }

View file

@ -306,11 +306,22 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
unsupported("addToStoreFromDump"); unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name, references), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = nar.first,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.references = references;
return info; return info;
})->path; })->path;
} }
@ -414,15 +425,22 @@ StorePath BinaryCacheStore::addToStore(
}); });
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, h, name, references), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
info.references = references;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info; return info;
})->path; })->path;
} }
@ -434,7 +452,7 @@ StorePath BinaryCacheStore::addTextToStore(
RepairFlag repair) RepairFlag repair)
{ {
auto textHash = hashString(htSHA256, s); auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references); auto path = makeTextPath(name, TextInfo { { textHash }, references });
if (!repair && isValidPath(path)) if (!repair && isValidPath(path))
return path; return path;
@ -443,10 +461,16 @@ StorePath BinaryCacheStore::addTextToStore(
dumpString(s, sink); dumpString(s, sink);
StringSource source(sink.s); StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first }; ValidPathInfo info {
*this,
std::string { name },
TextInfo {
{ .hash = textHash },
references,
},
nar.first,
};
info.narSize = nar.second; info.narSize = nar.second;
info.ca = TextHash { textHash };
info.references = references;
return info; return info;
})->path; })->path;
} }

View file

@ -2395,27 +2395,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
} }
}; };
auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { auto rewriteRefs = [&]() -> StoreReferences {
/* In the CA case, we need the rewritten refs to calculate the /* In the CA case, we need the rewritten refs to calculate the
final path, therefore we look for a *non-rewritten final path, therefore we look for a *non-rewritten
self-reference, and use a bool rather try to solve the self-reference, and use a bool rather try to solve the
computationally intractable fixed point. */ computationally intractable fixed point. */
std::pair<bool, StorePathSet> res { StoreReferences res {
false, .self = false,
{},
}; };
for (auto & r : references) { for (auto & r : references) {
auto name = r.name(); auto name = r.name();
auto origHash = std::string { r.hashPart() }; auto origHash = std::string { r.hashPart() };
if (r == *scratchPath) { if (r == *scratchPath) {
res.first = true; res.self = true;
} else if (auto outputRewrite = get(outputRewrites, origHash)) { } else if (auto outputRewrite = get(outputRewrites, origHash)) {
std::string newRef = *outputRewrite; std::string newRef = *outputRewrite;
newRef += '-'; newRef += '-';
newRef += name; newRef += name;
res.second.insert(StorePath { newRef }); res.others.insert(StorePath { newRef });
} else { } else {
res.second.insert(r); res.others.insert(r);
} }
} }
return res; return res;
@ -2448,18 +2447,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
break; break;
} }
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto refs = rewriteRefs(); ValidPathInfo newInfo0 {
worker.store,
auto finalPath = worker.store.makeFixedOutputPath(
outputHash.method,
got,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
refs.second, FixedOutputInfo {
refs.first); .hash = {
if (*scratchPath != finalPath) { .method = outputHash.method,
.hash = got,
},
.references = rewriteRefs(),
},
Hash::dummy,
};
if (*scratchPath != newInfo0.path) {
// Also rewrite the output path // Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) { auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
dumpPath(actualPath, rsink2); dumpPath(actualPath, rsink2);
rsink2.flush(); rsink2.flush();
}); });
@ -2470,19 +2473,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
} }
HashResult narHashAndSize = hashPath(htSHA256, actualPath); HashResult narHashAndSize = hashPath(htSHA256, actualPath);
ValidPathInfo newInfo0 { newInfo0.narHash = narHashAndSize.first;
finalPath,
narHashAndSize.first,
};
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
newInfo0.references = refs.second;
if (refs.first)
newInfo0.references.insert(newInfo0.path);
assert(newInfo0.ca); assert(newInfo0.ca);
return newInfo0; return newInfo0;
@ -2504,8 +2496,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
auto refs = rewriteRefs(); auto refs = rewriteRefs();
newInfo0.references = refs.second; newInfo0.references = std::move(refs.others);
if (refs.first) if (refs.self)
newInfo0.references.insert(newInfo0.path); newInfo0.references.insert(newInfo0.path);
return newInfo0; return newInfo0;
}, },
@ -2519,7 +2511,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* Check wanted hash */ /* Check wanted hash */
const Hash & wanted = dof.hash.hash; const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = getContentAddressHash(*newInfo0.ca); auto got = newInfo0.ca->getHash();
if (wanted != got) { if (wanted != got) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */

View file

@ -95,7 +95,9 @@ void PathSubstitutionGoal::tryNext()
subs.pop_front(); subs.pop_front();
if (ca) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); subPath = sub->makeFixedOutputPathFromCA(
std::string { storePath.name() },
ContentAddressWithReferences::withoutRefs(*ca));
if (sub->storeDir == worker.store.storeDir) if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath); assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) { } else if (sub->storeDir != worker.store.storeDir) {

View file

@ -9,7 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type); return makeFileIngestionPrefix(method) + printHashType(hash.type);
} }
std::string makeFileIngestionPrefix(const FileIngestionMethod m) std::string makeFileIngestionPrefix(FileIngestionMethod m)
{ {
switch (m) { switch (m) {
case FileIngestionMethod::Flat: case FileIngestionMethod::Flat:
@ -21,39 +21,35 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
} }
} }
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string ContentAddress::render() const
{
return "fixed:"
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true);
}
std::string renderContentAddress(ContentAddress ca)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash & th) { [](const TextHash & th) {
return "text:" + th.hash.to_string(Base32, true); return "text:"
+ th.hash.to_string(Base32, true);
}, },
[](FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash); return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
} }
}, ca); }, raw);
} }
std::string renderContentAddressMethod(ContentAddressMethod cam) std::string ContentAddressMethod::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHashMethod & th) { [](const TextHashMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(htSHA256);
}, },
[](FixedOutputHashMethod & fshm) { [](const FixedOutputHashMethod & fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
} }
}, cam); }, raw);
} }
/* /**
Parses content address strings up to the hash. * Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
{ {
@ -97,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
} }
ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddress ContentAddress::parse(std::string_view rawCa) {
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
@ -115,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
}); });
}, },
}, caMethod); }, caMethod.raw);
} }
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
{ {
std::string asPrefix = std::string{caMethod} + ":"; std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference // parseContentAddressMethodPrefix takes its argument by reference
@ -126,26 +122,55 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
return parseContentAddressMethodPrefix(asPrefixView); return parseContentAddressMethodPrefix(asPrefixView);
} }
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) std::optional<ContentAddress> ContentAddress::parseOpt(std::string_view rawCaOpt)
{ {
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); return rawCaOpt == ""
? std::nullopt
: std::optional { ContentAddress::parse(rawCaOpt) };
}; };
std::string renderContentAddress(std::optional<ContentAddress> ca) std::string renderContentAddress(std::optional<ContentAddress> ca)
{ {
return ca ? renderContentAddress(*ca) : ""; return ca ? ca->render() : "";
} }
Hash getContentAddressHash(const ContentAddress & ca) const Hash & ContentAddress::getHash() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextHash & th) { [](const TextHash & th) -> auto & {
return th.hash; return th.hash;
}, },
[](const FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) -> auto & {
return fsh.hash; return fsh.hash;
},
}, raw);
} }
}, ca);
bool StoreReferences::empty() const
{
return !self && others.empty();
}
size_t StoreReferences::size() const
{
return (self ? 1 : 0) + others.size();
}
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo {
.hash = h,
.references = {},
};
},
[&](const FixedOutputHash & h) -> ContentAddressWithReferences {
return FixedOutputInfo {
.hash = h,
.references = {},
};
},
}, ca.raw);
} }
} }

View file

@ -3,12 +3,30 @@
#include <variant> #include <variant>
#include "hash.hh" #include "hash.hh"
#include "path.hh"
#include "comparator.hh" #include "comparator.hh"
namespace nix { namespace nix {
/*
* Content addressing method
*/
/* We only have one way to hash text with references, so this is a single-value
type, mainly useful with std::variant.
*/
/** /**
* An enumeration of the ways we can serialize file system objects. * The single way we can serialize "text" file system objects.
*
* Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently.
*/
struct TextHashMethod : std::monostate { };
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/ */
enum struct FileIngestionMethod : uint8_t { enum struct FileIngestionMethod : uint8_t {
/** /**
@ -22,6 +40,53 @@ enum struct FileIngestionMethod : uint8_t {
Recursive = true Recursive = true
}; };
/**
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested.
*/
std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
};
/**
* An enumeration of all the ways we can serialize file system objects.
*
* Just the type of a content address. Combine with the hash itself, and
* we have a `ContentAddress` as defined below. Combine that, in turn,
* with info on references, and we have `ContentAddressWithReferences`,
* as defined further below.
*/
struct ContentAddressMethod
{
typedef std::variant<
TextHashMethod,
FixedOutputHashMethod
> Raw;
Raw raw;
GENERATE_CMP(ContentAddressMethod, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressMethod(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
static ContentAddressMethod parse(std::string_view rawCaMethod);
std::string render() const;
};
/*
* Mini content address
*/
/** /**
* Somewhat obscure, used by \ref Derivation derivations and * Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently. * `builtins.toFile` currently.
@ -36,7 +101,7 @@ struct TextHash {
}; };
/** /**
* For path computed by makeFixedOutputPath. * Used by most store objects that are content-addressed.
*/ */
struct FixedOutputHash { struct FixedOutputHash {
/** /**
@ -65,41 +130,96 @@ struct FixedOutputHash {
* - fixed:<r?>:<ht>:<h>: For paths computed by * - fixed:<r?>:<ht>:<h>: For paths computed by
* Store::makeFixedOutputPath() / Store::addToStore(). * Store::makeFixedOutputPath() / Store::addToStore().
*/ */
struct ContentAddress
{
typedef std::variant< typedef std::variant<
TextHash, TextHash,
FixedOutputHash FixedOutputHash
> ContentAddress; > Raw;
/** Raw raw;
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested. GENERATE_CMP(ContentAddress, me->raw);
*/
std::string makeFileIngestionPrefix(const FileIngestionMethod m); /* The moral equivalent of `using Raw::Raw;` */
ContentAddress(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/** /**
* Compute the content-addressability assertion (ValidPathInfo::ca) for * Compute the content-addressability assertion (ValidPathInfo::ca) for
* paths created by Store::makeFixedOutputPath() / Store::addToStore(). * paths created by Store::makeFixedOutputPath() / Store::addToStore().
*/ */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); std::string render() const;
std::string renderContentAddress(ContentAddress ca); static ContentAddress parse(std::string_view rawCa);
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
const Hash & getHash() const;
};
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
/* /*
We only have one way to hash text with references, so this is single-value * Full content address
type is only useful in std::variant. *
* See the schema for store paths in store-api.cc
*/ */
struct TextHashMethod { };
struct FixedOutputHashMethod { /**
FileIngestionMethod fileIngestionMethod; * A set of references to other store objects.
HashType hashType; *
* References to other store objects are tracked with store paths, self
* references however are tracked with a boolean.
*/
struct StoreReferences {
/**
* References to other store objects
*/
StorePathSet others;
/**
* Reference to this store object
*/
bool self = false;
/**
* @return true iff no references, i.e. others is empty and self is
* false.
*/
bool empty() const;
/**
* Returns the numbers of references, i.e. the size of others + 1
* iff self is true.
*/
size_t size() const;
GENERATE_CMP(StoreReferences, me->self, me->others);
};
// This matches the additional info that we need for makeTextPath
struct TextInfo {
TextHash hash;
/**
* References to other store objects only; self references
* disallowed
*/
StorePathSet references;
GENERATE_CMP(TextInfo, me->hash, me->references);
};
struct FixedOutputInfo {
FixedOutputHash hash;
/**
* References to other store objects or this one.
*/
StoreReferences references;
GENERATE_CMP(FixedOutputInfo, me->hash, me->references);
}; };
/** /**
@ -107,13 +227,27 @@ struct FixedOutputHashMethod {
* *
* A ContentAddress without a Hash. * A ContentAddress without a Hash.
*/ */
struct ContentAddressWithReferences
{
typedef std::variant< typedef std::variant<
TextHashMethod, TextInfo,
FixedOutputHashMethod FixedOutputInfo
> ContentAddressMethod; > Raw;
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); Raw raw;
std::string renderContentAddressMethod(ContentAddressMethod caMethod); GENERATE_CMP(ContentAddressWithReferences, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressWithReferences(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/**
* Create a ContentAddressWithReferences from a mere ContentAddress, by
* assuming no references in all cases.
*/
static ContentAddressWithReferences withoutRefs(const ContentAddress &);
};
} }

View file

@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod &) { [&](const TextHashMethod &) {
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](FixedOutputHashMethod & fohm) { [&](const FixedOutputHashMethod & fohm) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod); }, contentAddressMethod.raw);
}(); }();
logger->stopWork(); logger->stopWork();
@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
from >> info.registrationTime >> info.narSize >> info.ultimate; from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from); info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from)); info.ca = ContentAddress::parseOpt(readString(from));
from >> repair >> dontCheckSigs; from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs) if (!trusted && dontCheckSigs)
dontCheckSigs = false; dontCheckSigs = false;

View file

@ -36,8 +36,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{ {
return store.makeFixedOutputPath( return store.makeFixedOutputPath(
hash.method, hash.hash, outputPathName(drvName, outputName),
outputPathName(drvName, outputName)); { hash, {} });
} }
@ -942,7 +942,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {

View file

@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
throw Error("NAR hash is now mandatory"); throw Error("NAR hash is now mandatory");
info->narHash = Hash::parseAnyPrefixed(s); info->narHash = Hash::parseAnyPrefixed(s);
} }
info->ca = parseContentAddressOpt(readString(conn->from)); info->ca = ContentAddress::parseOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from); info->sigs = readStrings<StringSet>(conn->from);
auto s = readString(conn->from); auto s = readString(conn->from);

View file

@ -710,6 +710,7 @@ void canonicalisePathMetaData(const Path & path,
canonicalisePathMetaData(path, uidRange, inodesSeen); canonicalisePathMetaData(path, uidRange, inodesSeen);
} }
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{ {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
@ -888,7 +889,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
if (s) info->sigs = tokenizeString<StringSet>(s, " "); if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s); if (s) info->ca = ContentAddress::parseOpt(s);
/* Get the references. */ /* Get the references. */
auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
@ -1221,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
printStorePath(info.path), info.narSize, hashResult.second); printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) { if (info.ca) {
if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) { if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
auto actualFoHash = hashCAPath( auto actualFoHash = hashCAPath(
foHash->method, foHash->method,
foHash->hash.type, foHash->hash.type,
@ -1234,7 +1235,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
actualFoHash.hash.to_string(Base32, true)); actualFoHash.hash.to_string(Base32, true));
} }
} }
if (auto textHash = std::get_if<TextHash>(&*info.ca)) { if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath)); auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) { if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
@ -1320,7 +1321,19 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto [hash, size] = hashSink->finish(); auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name, references); ContentAddressWithReferences desc = FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
};
auto dstPath = makeFixedOutputPathFromCA(name, desc);
addTempRoot(dstPath); addTempRoot(dstPath);
@ -1364,10 +1377,13 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
optimisePath(realPath, repair); optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash.first }; ValidPathInfo info {
*this,
name,
std::move(desc),
narHash.first
};
info.narSize = narHash.second; info.narSize = narHash.second;
info.references = references;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info); registerValidPath(info);
} }
@ -1384,7 +1400,10 @@ StorePath LocalStore::addTextToStore(
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
auto hash = hashString(htSHA256, s); auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references); auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
});
addTempRoot(dstPath); addTempRoot(dstPath);

View file

@ -27,18 +27,17 @@ std::map<StorePath, StorePath> makeContentAddressed(
StringMap rewrites; StringMap rewrites;
StorePathSet references; StoreReferences refs;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) { for (auto & ref : oldInfo->references) {
if (ref == path) if (ref == path)
hasSelfReference = true; refs.self = true;
else { else {
auto i = remappings.find(ref); auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref; auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths? // FIXME: warn about unremapped paths?
if (replacement != ref) if (replacement != ref)
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
references.insert(std::move(replacement)); refs.others.insert(std::move(replacement));
} }
} }
@ -49,24 +48,28 @@ std::map<StorePath, StorePath> makeContentAddressed(
auto narModuloHash = hashModuloSink.finish().first; auto narModuloHash = hashModuloSink.finish().first;
auto dstPath = dstStore.makeFixedOutputPath( ValidPathInfo info {
FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); dstStore,
path.name(),
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
},
.references = std::move(refs),
},
Hash::dummy,
};
printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
StringSink sink2; StringSink sink2;
RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
rsink2(sink.s); rsink2(sink.s);
rsink2.flush(); rsink2.flush();
ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; info.narHash = hashString(htSHA256, sink2.s);
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
};
StringSource source(sink2.s); StringSource source(sink2.s);
dstStore.addToStore(info, source); dstStore.addToStore(info, source);

View file

@ -273,7 +273,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9)); narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig); narInfo->sigs.insert(sig);
narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11));
return {oValid, narInfo}; return {oValid, narInfo};
}); });

View file

@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "CA") { else if (name == "CA") {
if (ca) throw corrupt(); if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field? // FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value); ca = ContentAddress::parseOpt(value);
} }
pos = eol + 1; pos = eol + 1;

View file

@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
uint64_t fileSize = 0; uint64_t fileSize = 0;
NarInfo() = delete; NarInfo() = delete;
NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash)
: ValidPathInfo(store, std::move(name), std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence); NarInfo(const Store & store, const std::string & s, const std::string & whence);

View file

@ -21,25 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store))); sigs.insert(secretKey.signDetached(fingerprint(store)));
} }
std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const
bool ValidPathInfo::isContentAddressed(const Store & store) const
{ {
if (! ca) return false; if (! ca)
return std::nullopt;
auto caPath = std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & th) { [&](const TextHash & th) -> ContentAddressWithReferences {
return store.makeTextPath(path.name(), th.hash, references); assert(references.count(path) == 0);
return TextInfo {
.hash = th,
.references = references,
};
}, },
[&](const FixedOutputHash & fsh) { [&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
auto refs = references; auto refs = references;
bool hasSelfReference = false; bool hasSelfReference = false;
if (refs.count(path)) { if (refs.count(path)) {
hasSelfReference = true; hasSelfReference = true;
refs.erase(path); refs.erase(path);
} }
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); return FixedOutputInfo {
.hash = foh,
.references = {
.others = std::move(refs),
.self = hasSelfReference,
},
};
},
}, ca->raw);
} }
}, *ca);
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto fullCaOpt = contentAddressWithReferences();
if (! fullCaOpt)
return false;
auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt);
bool res = caPath == path; bool res = caPath == path;
@ -77,6 +97,29 @@ Strings ValidPathInfo::shortRefs() const
} }
ValidPathInfo::ValidPathInfo(
const Store & store,
std::string_view name,
ContentAddressWithReferences && ca,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(name, ca))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = std::move(ti.references);
this->ca = std::move((TextHash &&) ti);
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references.others);
if (foi.references.self)
this->references.insert(path);
this->ca = std::move((FixedOutputHash &&) foi);
},
}, std::move(ca).raw);
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{ {
return read(source, store, format, store.parseStorePath(readString(source))); return read(source, store, format, store.parseStorePath(readString(source)));
@ -93,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
if (format >= 16) { if (format >= 16) {
source >> info.ultimate; source >> info.ultimate;
info.sigs = readStrings<StringSet>(source); info.sigs = readStrings<StringSet>(source);
info.ca = parseContentAddressOpt(readString(source)); info.ca = ContentAddress::parseOpt(readString(source));
} }
return info; return info;
} }

View file

@ -92,6 +92,13 @@ struct ValidPathInfo
void sign(const Store & store, const SecretKey & secretKey); void sign(const Store & store, const SecretKey & secretKey);
/**
* @return The `ContentAddressWithReferences` that determines the
* store path for a content-addressed store object, `std::nullopt`
* for an input-addressed store object.
*/
std::optional<ContentAddressWithReferences> contentAddressWithReferences() const;
/** /**
* @return true iff the path is verifiably content-addressed. * @return true iff the path is verifiably content-addressed.
*/ */
@ -118,6 +125,9 @@ struct ValidPathInfo
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(const Store & store,
std::string_view name, ContentAddressWithReferences && ca, Hash narHash);
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
static ValidPathInfo read(Source & source, const Store & store, unsigned int format); static ValidPathInfo read(Source & source, const Store & store, unsigned int format);

View file

@ -78,7 +78,7 @@ void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & o
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{ {
return parseContentAddress(readString(from)); return ContentAddress::parse(readString(from));
} }
void write(const Store & store, Sink & out, const ContentAddress & ca) void write(const Store & store, Sink & out, const ContentAddress & ca)
@ -168,7 +168,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
{ {
return parseContentAddressOpt(readString(from)); return ContentAddress::parseOpt(readString(from));
} }
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
@ -586,7 +586,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< renderContentAddressMethod(caMethod); << caMethod.render();
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -644,7 +644,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
} }
} }
}, caMethod); }, caMethod.raw);
auto path = parseStorePath(readString(conn->from)); auto path = parseStorePath(readString(conn->from));
// Release our connection to prevent a deadlock in queryPathInfo(). // Release our connection to prevent a deadlock in queryPathInfo().
conn_.reset(); conn_.reset();

View file

@ -7,6 +7,7 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "url.hh" #include "url.hh"
#include "references.hh"
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
#include "remote-store.hh" #include "remote-store.hh"
@ -98,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
silly, but it's done that way for compatibility). <id> is the silly, but it's done that way for compatibility). <id> is the
name of the output (usually, "out"). name of the output (usually, "out").
<h2> = base-16 representation of a SHA-256 hash of: <h2> = base-16 representation of a SHA-256 hash of <s2>
<s2> =
if <type> = "text:...": if <type> = "text:...":
the string written to the resulting store path the string written to the resulting store path
if <type> = "source": if <type> = "source:...":
the serialisation of the path from which this store path is the serialisation of the path from which this store path is
copied, as returned by hashPath() copied, as returned by hashPath()
if <type> = "output:<id>": if <type> = "output:<id>":
@ -162,63 +165,63 @@ StorePath Store::makeOutputPath(std::string_view id,
} }
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in, say, <s2> (per the grammar above)
since that would be ambiguous. */
static std::string makeType( static std::string makeType(
const Store & store, const Store & store,
std::string && type, std::string && type,
const StorePathSet & references, const StoreReferences & references)
bool hasSelfReference = false)
{ {
for (auto & i : references) { for (auto & i : references.others) {
type += ":"; type += ":";
type += store.printStorePath(i); type += store.printStorePath(i);
} }
if (hasSelfReference) type += ":self"; if (references.self) type += ":self";
return std::move(type); return std::move(type);
} }
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name);
} else { } else {
assert(references.empty()); assert(info.references.size() == 0);
return makeStorePath("output:out", return makeStorePath("output:out",
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ makeFileIngestionPrefix(method) + makeFileIngestionPrefix(info.hash.method)
+ hash.to_string(Base16, true) + ":"), + info.hash.hash.to_string(Base16, true) + ":"),
name); name);
} }
} }
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
.self = false,
}),
info.hash.hash,
name);
}
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
{ {
// New template // New template
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & th) { [&](const TextInfo & ti) {
return makeTextPath(name, th.hash, references); return makeTextPath(name, ti);
}, },
[&](const FixedOutputHash & fsh) { [&](const FixedOutputInfo & foi) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); return makeFixedOutputPath(name, foi);
} }
}, ca); }, ca.raw);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType(*this, "text", references), hash, name);
} }
@ -228,7 +231,14 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
Hash h = method == FileIngestionMethod::Recursive Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath); : hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h); FixedOutputInfo caInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
} }
@ -237,7 +247,10 @@ StorePath Store::computeStorePathForText(
std::string_view s, std::string_view s,
const StorePathSet & references) const const StorePathSet & references) const
{ {
return makeTextPath(name, hashString(htSHA256, s), references); return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
});
} }
@ -425,11 +438,18 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info { ValidPathInfo info {
makeFixedOutputPath(method, hash, name), *this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {},
},
narHash, narHash,
}; };
info.narSize = narSize; info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = sinkToSource([&](Sink & scratchpadSink) {
@ -521,7 +541,9 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
// Recompute store path so that we can use a different store root. // Recompute store path so that we can use a different store root.
if (path.second) { if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); subPath = makeFixedOutputPathFromCA(
path.first.name(),
ContentAddressWithReferences::withoutRefs(*path.second));
if (sub->storeDir == storeDir) if (sub->storeDir == storeDir)
assert(subPath == path.first); assert(subPath == path.first);
if (subPath != path.first) if (subPath != path.first)
@ -538,10 +560,11 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.first, SubstitutablePathInfo{ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver, .deriver = info->deriver,
info->references, .references = info->references,
narInfo ? narInfo->fileSize : 0, .downloadSize = narInfo ? narInfo->fileSize : 0,
info->narSize}); .narSize = info->narSize,
});
} catch (InvalidPath &) { } catch (InvalidPath &) {
} catch (SubstituterDisabled &) { } catch (SubstituterDisabled &) {
} catch (Error & e) { } catch (Error & e) {
@ -1025,7 +1048,9 @@ void copyStorePath(
// recompute store path on the chance dstStore does it differently // recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info); auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); info2->path = dstStore.makeFixedOutputPathFromCA(
info->path.name(),
info->contentAddressWithReferences().value());
if (dstStore.storeDir == srcStore.storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(info->path == info2->path); assert(info->path == info2->path);
info = info2; info = info2;
@ -1137,7 +1162,9 @@ std::map<StorePath, StorePath> copyPaths(
auto storePathForSrc = currentPathInfo.path; auto storePathForSrc = currentPathInfo.path;
auto storePathForDst = storePathForSrc; auto storePathForDst = storePathForSrc;
if (currentPathInfo.ca && currentPathInfo.references.empty()) { if (currentPathInfo.ca && currentPathInfo.references.empty()) {
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); storePathForDst = dstStore.makeFixedOutputPathFromCA(
currentPathInfo.path.name(),
currentPathInfo.contentAddressWithReferences().value());
if (dstStore.storeDir == srcStore.storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePathForSrc); assert(storePathForDst == storePathForSrc);
if (storePathForDst != storePathForSrc) if (storePathForDst != storePathForSrc)

View file

@ -268,17 +268,11 @@ public:
StorePath makeOutputPath(std::string_view id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash, StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/** /**
* Preparatory part of addToStore(). * Preparatory part of addToStore().

View file

@ -204,10 +204,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */ /* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs) static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{ {
auto recursive = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
for (auto i : opFlags) for (auto i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive; if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i); else throw UsageError("unknown flag '%1%'", i);
if (opArgs.size() != 3) if (opArgs.size() != 3)
@ -218,7 +218,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
std::string hash = *i++; std::string hash = *i++;
std::string name = *i++; std::string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
.references = {},
})));
} }
@ -964,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs)
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
in >> info.registrationTime >> info.narSize >> info.ultimate; in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in); info.sigs = readStrings<StringSet>(in);
info.ca = parseContentAddressOpt(readString(in)); info.ca = ContentAddress::parseOpt(readString(in));
if (info.narSize == 0) if (info.narSize == 0)
throw Error("narInfo is too old and missing the narSize field"); throw Error("narInfo is too old and missing the narSize field");

View file

@ -42,14 +42,18 @@ struct CmdAddToStore : MixDryRun, StoreCommand
} }
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(ingestionMethod, hash, *namePart), *store,
std::move(*namePart),
FixedOutputInfo {
.hash = {
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
.references = {},
},
narHash, narHash,
}; };
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = std::optional { FixedOutputHash {
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) { if (!dryRun) {
auto source = StringSource(sink.s); auto source = StringSource(sink.s);

View file

@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile(
the store. */ the store. */
if (expectedHash) { if (expectedHash) {
hashType = expectedHash->type; hashType = expectedHash->type;
storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
.hash = {
.method = ingestionMethod,
.hash = *expectedHash,
},
.references = {},
});
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = expectedHash; hash = expectedHash;
else else
@ -118,7 +124,7 @@ std::tuple<StorePath, Hash> prefetchFile(
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
storePath = info.path; storePath = info.path;
assert(info.ca); assert(info.ca);
hash = getContentAddressHash(*info.ca); hash = info.ca->getHash();
} }
return {storePath.value(), hash.value()}; return {storePath.value(), hash.value()};

View file

@ -200,12 +200,22 @@ struct ProfileManifest
auto narHash = hashString(htSHA256, sink.s); auto narHash = hashString(htSHA256, sink.s);
ValidPathInfo info { ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), *store,
"profile",
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
.references = {
.others = std::move(references),
// profiles never refer to themselves
.self = false,
},
},
narHash, narHash,
}; };
info.references = std::move(references);
info.narSize = sink.s.size(); info.narSize = sink.s.size();
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
StringSource source(sink.s); StringSource source(sink.s);
store->addToStore(info, source); store->addToStore(info, source);