Derivations can output "text-hashed" data

In particular, this means that derivations can output derivations. But
that ramification isn't (yet!) useful as we would want, since there is
no way to have a dependent derivation that is itself a dependent
derivation.
This commit is contained in:
John Ericson 2020-10-12 23:51:23 +00:00
parent a0f369aa3f
commit a4e5de1b9d
15 changed files with 322 additions and 137 deletions

View file

@ -851,7 +851,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
bool contentAddressed = false; bool contentAddressed = false;
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat; ContentAddressMethod ingestionMethod = FileIngestionMethod::Flat;
StringSet outputs; StringSet outputs;
outputs.insert("out"); outputs.insert("out");
@ -864,6 +864,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto handleHashMode = [&](const std::string & s) { auto handleHashMode = [&](const std::string & s) {
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else if (s == "text") ingestionMethod = TextHashMethod {};
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
@ -995,8 +996,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
for (auto & j : refs) { for (auto & j : refs) {
drv.inputSrcs.insert(j); drv.inputSrcs.insert(j);
if (j.isDerivation()) if (j.isDerivation()) {
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); Derivation jDrv = state.store->readDerivation(j);
if(jDrv.type() != DerivationType::CAFloating)
drv.inputDrvs[j] = jDrv.outputNames();
}
} }
} }
@ -1025,9 +1029,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}); });
/* Check whether the derivation name is valid. */ /* Check whether the derivation name is valid. */
if (isDerivation(drvName)) if (isDerivation(drvName) && ingestionMethod != ContentAddressMethod { TextHashMethod { } })
throw EvalError({ throw EvalError({
.hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .hint = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
.errPos = posDrvName .errPos = posDrvName
}); });
@ -1045,22 +1049,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht); Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { // FIXME non-trivial fixed refs set
{ auto ca = contentAddressFromMethodHashAndRefs(
.method = ingestionMethod, ingestionMethod,
.hash = h, std::move(h),
}, {});
{},
}); DerivationOutputCAFixed dof { .ca = ca };
drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
.output = DerivationOutputCAFixed { drv.outputs.insert_or_assign("out", DerivationOutput { .output = dof });
.hash = FixedOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
},
},
});
} }
else if (contentAddressed) { else if (contentAddressed) {

View file

@ -4070,19 +4070,26 @@ void DerivationGoal::registerOutputs()
auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName); auto & st = outputStats.at(outputName);
if (outputHash.method == FileIngestionMethod::Flat) { if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextHashMethod {} })
{
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
"output path '%1%' should be a non-executable regular file " "output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)", "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
actualPath); actualPath);
} }
rewriteOutput(); rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */ /* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath.hashPart() }; std::string oldHashPart { scratchPath.hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart }; HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) { std::visit(overloaded {
[&](TextHashMethod _) {
readFile(actualPath, caSink);
},
[&](FileIngestionMethod m2) {
switch (m2) {
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink); dumpPath(actualPath, caSink);
break; break;
@ -4090,6 +4097,8 @@ void DerivationGoal::registerOutputs()
readFile(actualPath, caSink); readFile(actualPath, caSink);
break; break;
} }
},
}, outputHash.method);
auto got = caSink.finish().first; auto got = caSink.finish().first;
HashModuloSink narSink { htSHA256, oldHashPart }; HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink); dumpPath(actualPath, narSink);
@ -4098,13 +4107,10 @@ void DerivationGoal::registerOutputs()
worker.store, worker.store,
{ {
.name = outputPathName(drv->name, outputName), .name = outputPathName(drv->name, outputName),
.info = FixedOutputInfo { .info = contentAddressFromMethodHashAndRefs(
{ outputHash.method,
.method = outputHash.method, std::move(got),
.hash = got, rewriteRefs()),
},
rewriteRefs(),
},
}, },
narHashAndSize.first, narHashAndSize.first,
}; };
@ -4132,13 +4138,14 @@ void DerivationGoal::registerOutputs()
return newInfo0; return newInfo0;
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
auto wanted = getContentAddressHash(dof.ca);
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
.method = dof.hash.method, .method = getContentAddressMethod(dof.ca),
.hashType = dof.hash.hash.type, .hashType = wanted.type,
}); });
/* Check wanted hash */ /* Check wanted hash */
Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = getContentAddressHash(*newInfo0.ca); auto got = getContentAddressHash(*newInfo0.ca);
if (wanted != got) { if (wanted != got) {
@ -4151,6 +4158,11 @@ void DerivationGoal::registerOutputs()
wanted.to_string(SRI, true), wanted.to_string(SRI, true),
got.to_string(SRI, true))); got.to_string(SRI, true)));
} }
if (static_cast<const PathReferences<StorePath> &>(newInfo0) != PathReferences<StorePath> {})
delayedException = std::make_exception_ptr(
BuildError("illegal path references in fixed-output derivation '%s'",
worker.store.printStorePath(drvPath)));
return newInfo0; return newInfo0;
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {

View file

@ -10,7 +10,7 @@ std::string FixedOutputHash::printMethodAlgo() const
} }
std::string makeFileIngestionPrefix(const FileIngestionMethod m) std::string makeFileIngestionPrefix(FileIngestionMethod m)
{ {
switch (m) { switch (m) {
case FileIngestionMethod::Flat: case FileIngestionMethod::Flat:
@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
assert(false); assert(false);
} }
std::string makeContentAddressingPrefix(ContentAddressMethod m) {
return std::visit(overloaded {
[](TextHashMethod _) -> std::string { return "text:"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2);
},
}, m);
}
ContentAddressMethod parseContentAddressingPrefix(std::string_view & m)
{
ContentAddressMethod method = FileIngestionMethod::Flat;
if (splitPrefix(m, "r:"))
method = FileIngestionMethod::Recursive;
else if (splitPrefix(m, "text:"))
method = TextHashMethod {};
return method;
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{ {
return "fixed:" return "fixed:"
@ -43,14 +64,14 @@ std::string renderContentAddress(ContentAddress ca)
}, ca); }, ca);
} }
std::string renderContentAddressMethod(ContentAddressMethod cam) std::string renderContentAddressMethodAndHash(ContentAddressMethod cam, HashType ht)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHashMethod &th) { [&](TextHashMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(ht);
}, },
[](FixedOutputHashMethod &fshm) { [&](FileIngestionMethod & fim) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
} }
}, cam); }, cam);
} }
@ -58,7 +79,7 @@ std::string renderContentAddressMethod(ContentAddressMethod cam)
/* /*
Parses content address strings up to the hash. Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
{ {
std::string_view wholeInput { rest }; std::string_view wholeInput { rest };
@ -82,19 +103,19 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
if (prefix == "text") { if (prefix == "text") {
// No parsing of the ingestion method, "text" only support flat. // No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
if (hashType != htSHA256) return {
throw Error("text content address hash should use %s, but instead uses %s", TextHashMethod {},
printHashType(htSHA256), printHashType(hashType)); std::move(hashType),
return TextHashMethod {}; };
} else if (prefix == "fixed") { } else if (prefix == "fixed") {
// Parse method // Parse method
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:")) if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
return FixedOutputHashMethod { return {
.fileIngestionMethod = method, std::move(method),
.hashType = std::move(hashType), std::move(hashType),
}; };
} else } else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
@ -103,25 +124,24 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddress parseContentAddress(std::string_view rawCa) {
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
return std::visit( return std::visit(overloaded {
overloaded { [&](TextHashMethod _) {
[&](TextHashMethod thm) {
return ContentAddress(TextHash { return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
}); });
}, },
[&](FixedOutputHashMethod fohMethod) { [&](FileIngestionMethod fim) {
return ContentAddress(FixedOutputHash { return ContentAddress(FixedOutputHash {
.method = fohMethod.fileIngestionMethod, .method = fim,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
}); });
}, },
}, caMethod); }, caMethod);
} }
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) std::pair<ContentAddressMethod, HashType> parseContentAddressMethod(std::string_view caMethod)
{ {
std::string_view asPrefix {std::string{caMethod} + ":"}; std::string_view asPrefix {std::string{caMethod} + ":"};
return parseContentAddressMethodPrefix(asPrefix); return parseContentAddressMethodPrefix(asPrefix);
@ -137,6 +157,42 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
return ca ? renderContentAddress(*ca) : ""; return ca ? renderContentAddress(*ca) : "";
} }
ContentAddressWithReferences contentAddressFromMethodHashAndRefs(
ContentAddressMethod method, Hash && hash, PathReferences<StorePath> && refs)
{
return std::visit(overloaded {
[&](TextHashMethod _) -> ContentAddressWithReferences {
if (refs.hasSelfReference)
throw UsageError("Cannot have a self reference with text hashing scheme");
return TextInfo {
{ .hash = std::move(hash) },
std::move(refs.references),
};
},
[&](FileIngestionMethod m2) -> ContentAddressWithReferences {
return FixedOutputInfo {
{
.method = m2,
.hash = std::move(hash),
},
std::move(refs),
};
},
}, method);
}
ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences & ca)
{
return std::visit(overloaded {
[](TextInfo th) -> ContentAddressMethod {
return TextHashMethod {};
},
[](FixedOutputInfo fsh) -> ContentAddressMethod {
return fsh.method;
},
}, ca);
}
Hash getContentAddressHash(const ContentAddress & ca) Hash getContentAddressHash(const ContentAddress & ca)
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -160,4 +216,21 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
}, ca); }, ca);
} }
Hash getContentAddressHash(const ContentAddressWithReferences & ca)
{
return std::visit(overloaded {
[](TextInfo th) {
return th.hash;
},
[](FixedOutputInfo fsh) {
return fsh.hash;
},
}, ca);
}
std::string printMethodAlgo(const ContentAddressWithReferences & ca) {
return makeContentAddressingPrefix(getContentAddressMethod(ca))
+ printHashType(getContentAddressHash(ca).type);
}
} }

View file

@ -10,11 +10,45 @@ namespace nix {
* Mini content address * Mini content address
*/ */
/* We only have one way to hash text with references, so this is a single-value
type, mainly useful with std::variant.
*/
struct TextHashMethod : std::monostate { };
enum struct FileIngestionMethod : uint8_t { enum struct FileIngestionMethod : uint8_t {
Flat = false, Flat = false,
Recursive = true Recursive = true
}; };
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(FileIngestionMethod m);
/* Just the type of a content address. Combine with the hash itself, and we
have a `ContentAddress` as defined below. Combine that, in turn, with info
on references, and we have `ContentAddressWithReferences`, as defined
further below. */
typedef std::variant<
TextHashMethod,
FileIngestionMethod
> ContentAddressMethod;
/* Parse and pretty print the algorithm which indicates how the files
were ingested, with the the fixed output case not prefixed for back
compat. */
std::string makeContentAddressingPrefix(ContentAddressMethod m);
ContentAddressMethod parseContentAddressingPrefix(std::string_view & m);
/* Parse and pretty print a content addressing method and hash in a
nicer way, prefixing both cases. */
std::string renderContentAddressMethodAndHash(ContentAddressMethod cam, HashType ht);
std::pair<ContentAddressMethod, HashType> parseContentAddressMethod(std::string_view caMethod);
struct TextHash { struct TextHash {
Hash hash; Hash hash;
@ -27,6 +61,7 @@ struct FixedOutputHash {
std::string printMethodAlgo() const; std::string printMethodAlgo() const;
}; };
/* /*
We've accumulated several types of content-addressed paths over the years; We've accumulated several types of content-addressed paths over the years;
fixed-output derivations support multiple hash algorithms and serialisation fixed-output derivations support multiple hash algorithms and serialisation
@ -43,10 +78,6 @@ typedef std::variant<
FixedOutputHash // for path computed by makeFixedOutputPath FixedOutputHash // for path computed by makeFixedOutputPath
> ContentAddress; > ContentAddress;
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(ContentAddress ca);
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -57,24 +88,6 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca); Hash getContentAddressHash(const ContentAddress & ca);
/*
We only have one way to hash text with references, so this is single-value
type is only useful in std::variant.
*/
struct TextHashMethod { };
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
};
typedef std::variant<
TextHashMethod,
FixedOutputHashMethod
> ContentAddressMethod;
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
/* /*
* References set * References set
@ -92,6 +105,12 @@ struct PathReferences
&& hasSelfReference == other.hasSelfReference; && hasSelfReference == other.hasSelfReference;
} }
bool operator != (const PathReferences<Ref> & other) const
{
return references != other.references
|| hasSelfReference != other.hasSelfReference;
}
/* Functions to view references + hasSelfReference as one set, mainly for /* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */ compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const; StorePathSet referencesPossiblyToSelf(const Ref & self) const;
@ -151,6 +170,14 @@ typedef std::variant<
ContentAddressWithReferences caWithoutRefs(const ContentAddress &); ContentAddressWithReferences caWithoutRefs(const ContentAddress &);
ContentAddressWithReferences contentAddressFromMethodHashAndRefs(
ContentAddressMethod method, Hash && hash, PathReferences<StorePath> && refs);
ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences & ca);
Hash getContentAddressHash(const ContentAddressWithReferences & ca);
std::string printMethodAlgo(const ContentAddressWithReferences &);
struct StorePathDescriptor { struct StorePathDescriptor {
std::string name; std::string name;
ContentAddressWithReferences info; ContentAddressWithReferences info;

View file

@ -380,20 +380,23 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); auto [contentAddressMethod, hashType] = parseContentAddressMethod(camStr);
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod &_) { [&](TextHashMethod _) {
if (hashType != htSHA256)
throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given",
printHashType(hashType));
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](FixedOutputHashMethod &fohm) { [&](FileIngestionMethod fim) {
if (!refs.empty()) if (!refs.empty())
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); auto path = store->addToStoreFromDump(source, name, fim, hashType, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod); }, contentAddressMethod);

View file

@ -2,6 +2,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include "split.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
@ -26,9 +27,10 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath( return store.makeFixedOutputPathFromCA(StorePathDescriptor {
outputPathName(drvName, outputName), .name = outputPathName(drvName, outputName),
{ hash, {} }); .info = ca,
});
} }
@ -150,23 +152,19 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, static DerivationOutput parseDerivationOutput(const Store & store,
std::string_view pathS, std::string_view hashAlgo, std::string_view hash) std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
{ {
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; ContentAddressMethod method = parseContentAddressingPrefix(hashAlgo);
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash != "") { if (hashS != "") {
validatePath(pathS); validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputCAFixed { .output = DerivationOutputCAFixed {
.hash = FixedOutputHash { // FIXME non-trivial fixed refs set
.method = std::move(method), .ca = contentAddressFromMethodHashAndRefs(
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), method, std::move(hash), {}),
},
}, },
}; };
} else { } else {
@ -317,12 +315,12 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, printMethodAlgo(dof.ca));
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, getContentAddressHash(dof.ca).to_string(Base16, false));
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
}, i.second.output); }, i.second.output);
@ -482,8 +480,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutputCAFixed>(i.second.output); auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":" + printMethodAlgo(dof.ca) + ":"
+ dof.hash.hash.to_string(Base16, false) + ":" + getContentAddressHash(dof.ca).to_string(Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
@ -612,12 +610,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first)) out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo() << printMethodAlgo(dof.ca)
<< dof.hash.hash.to_string(Base16, false); << getContentAddressHash(dof.ca).to_string(Base16, false);
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
out << "" out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << (makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType))
<< ""; << "";
}, },
}, i.second.output); }, i.second.output);

View file

@ -27,7 +27,7 @@ struct DerivationOutputInputAddressed
according to that fixed output. */ according to that fixed output. */
struct DerivationOutputCAFixed struct DerivationOutputCAFixed
{ {
FixedOutputHash hash; /* hash used for expected hash computation */ ContentAddressWithReferences ca; /* hash and refs used for validating output */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
}; };
@ -37,7 +37,7 @@ struct DerivationOutputCAFixed
struct DerivationOutputCAFloating struct DerivationOutputCAFloating
{ {
/* information used for expected hash computation */ /* information used for expected hash computation */
FileIngestionMethod method; ContentAddressMethod method;
HashType hashType; HashType hashType;
}; };

View file

@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); auto path = dof.path(*this, drvName, i.first);
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](DerivationOutputCAFloating _) { [&](DerivationOutputCAFloating _) {

View file

@ -111,9 +111,21 @@ void Store::computeFSClosure(const StorePath & startPath,
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{ {
auto out = drv.outputs.find("out"); auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) { if (out == drv.outputs.end())
if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output)) return std::nullopt;
return v->hash; if (auto dof = std::get_if<DerivationOutputCAFixed>(&out->second.output)) {
return std::visit(overloaded {
[&](TextInfo ti) -> std::optional<ContentAddress> {
if (!ti.references.empty())
return std::nullopt;
return static_cast<TextHash>(ti);
},
[&](FixedOutputInfo fi) -> std::optional<ContentAddress> {
if (fi.references != PathReferences<StorePath> {})
return std::nullopt;
return static_cast<FixedOutputHash>(fi);
},
}, dof->ca);
} }
return std::nullopt; return std::nullopt;
} }

View file

@ -457,6 +457,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
Source & dump, Source & dump,
const string & name, const string & name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair) RepairFlag repair)
{ {
@ -468,7 +469,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< renderContentAddressMethod(caMethod); << renderContentAddressMethodAndHash(caMethod, hashType);
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -484,18 +485,21 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
std::visit(overloaded { std::visit(overloaded {
[&](TextHashMethod thm) -> void { [&](TextHashMethod thm) -> void {
if (hashType != htSHA256)
throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given",
printHashType(hashType));
std::string s = dump.drain(); std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s; conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn.processStderr(); conn.processStderr();
}, },
[&](FixedOutputHashMethod fohm) -> void { [&](FileIngestionMethod fim) -> void {
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) << (fim == FileIngestionMethod::Recursive ? 1 : 0)
<< printHashType(fohm.hashType); << printHashType(hashType);
try { try {
conn->to.written = 0; conn->to.written = 0;
@ -503,7 +507,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
connections->incCapacity(); connections->incCapacity();
{ {
Finally cleanup([&]() { connections->decCapacity(); }); Finally cleanup([&]() { connections->decCapacity(); });
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { if (fim == FileIngestionMethod::Recursive) {
dump.drainInto(conn->to); dump.drainInto(conn->to);
} else { } else {
std::string contents = dump.drain(); std::string contents = dump.drain();
@ -536,7 +540,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashType, RepairFlag repair) FileIngestionMethod method, HashType hashType, RepairFlag repair)
{ {
StorePathSet references; StorePathSet references;
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; return addCAToStore(dump, name, method, hashType, references, repair)->path;
} }
@ -597,7 +601,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
StringSource source(s); StringSource source(s);
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; return addCAToStore(source, name, TextHashMethod {}, htSHA256, references, repair)->path;
} }

View file

@ -68,6 +68,7 @@ public:
Source & dump, Source & dump,
const string & name, const string & name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair); RepairFlag repair);

View file

@ -76,11 +76,12 @@ struct CmdShowDerivation : InstallablesCommand
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName)));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hashAlgo", printMethodAlgo(dof.ca));
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); outputObj.attr("hash", getContentAddressHash(dof.ca).to_string(Base16, false));
// FIXME print refs?
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); outputObj.attr("hashAlgo", makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType));
}, },
}, output.output); }, output.output);
} }

View file

@ -35,7 +35,8 @@ nix_tests = \
recursive.sh \ recursive.sh \
describe-stores.sh \ describe-stores.sh \
flakes.sh \ flakes.sh \
content-addressed.sh content-addressed.sh \
text-hashed-output.sh
# parallel.sh # parallel.sh
# build-remote-content-addressed-fixed.sh \ # build-remote-content-addressed-fixed.sh \

View file

@ -0,0 +1,29 @@
with import ./config.nix;
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
rec {
root = mkDerivation {
name = "text-hashed-root";
buildCommand = ''
set -x
echo "Building a CA derivation"
mkdir -p $out
echo "Hello World" > $out/hello
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
dependent = mkDerivation {
name = "text-hashed-root.drv";
buildCommand = ''
echo "Copying the derivation"
cp ${root.drvPath} $out
'';
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
};
}

View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
source common.sh
# In the corresponding nix file, we have two derivations: the first, named root,
# is a normal recursive derivation, while the second, named dependent, has the
# new outputHashMode "text". Note that in "dependent", we don't refer to the
# build output of root, but only to the path of the drv file. For this reason,
# we only need to:
#
# - instantiate the root derivation
# - build the dependent derivation
# - check that the path of the output coincides with that of the original derivation
drv=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A root)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv"
drvDep=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A dependent)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drvDep"
out1=$(nix-build --experimental-features ca-derivations ./text-hashed-output.nix -A dependent --no-out-link)
nix --experimental-features 'nix-command ca-derivations' path-info $drv --derivation --json | jq
nix --experimental-features 'nix-command ca-derivations' path-info $out1 --derivation --json | jq
test $out1 == $drv