From a4e5de1b9d26584615946057430df9e63d842f53 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Oct 2020 23:51:23 +0000 Subject: [PATCH] Derivations can output "text-hashed" data In particular, this means that derivations can output derivations. But that ramification isn't (yet!) useful as we would want, since there is no way to have a dependent derivation that is itself a dependent derivation. --- src/libexpr/primops.cc | 40 +++++----- src/libstore/build.cc | 52 ++++++++----- src/libstore/content-address.cc | 133 +++++++++++++++++++++++++------- src/libstore/content-address.hh | 71 +++++++++++------ src/libstore/daemon.cc | 11 ++- src/libstore/derivations.cc | 42 +++++----- src/libstore/derivations.hh | 4 +- src/libstore/local-store.cc | 2 +- src/libstore/misc.cc | 18 ++++- src/libstore/remote-store.cc | 20 +++-- src/libstore/remote-store.hh | 1 + src/nix/show-derivation.cc | 7 +- tests/local.mk | 3 +- tests/text-hashed-output.nix | 29 +++++++ tests/text-hashed-output.sh | 26 +++++++ 15 files changed, 322 insertions(+), 137 deletions(-) create mode 100644 tests/text-hashed-output.nix create mode 100644 tests/text-hashed-output.sh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c74b67658..9d49eeef3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -851,7 +851,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * bool contentAddressed = false; std::optional outputHash; std::string outputHashAlgo; - auto ingestionMethod = FileIngestionMethod::Flat; + ContentAddressMethod ingestionMethod = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); @@ -864,6 +864,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto handleHashMode = [&](const std::string & s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; + else if (s == "text") ingestionMethod = TextHashMethod {}; else throw EvalError({ .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), @@ -995,8 +996,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) { drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + if (j.isDerivation()) { + Derivation jDrv = state.store->readDerivation(j); + if(jDrv.type() != DerivationType::CAFloating) + drv.inputDrvs[j] = jDrv.outputNames(); + } } } @@ -1025,9 +1029,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); /* Check whether the derivation name is valid. */ - if (isDerivation(drvName)) + if (isDerivation(drvName) && ingestionMethod != ContentAddressMethod { TextHashMethod { } }) throw EvalError({ - .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .hint = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), .errPos = posDrvName }); @@ -1045,22 +1049,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - { - .method = ingestionMethod, - .hash = h, - }, - {}, - }); - drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, - }, - }); + // FIXME non-trivial fixed refs set + auto ca = contentAddressFromMethodHashAndRefs( + ingestionMethod, + std::move(h), + {}); + + DerivationOutputCAFixed dof { .ca = ca }; + + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = dof }); } else if (contentAddressed) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 48e123f36..a23c7f333 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4070,26 +4070,35 @@ void DerivationGoal::registerOutputs() auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { + if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || + outputHash.method == ContentAddressMethod { TextHashMethod {} }) + { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } rewriteOutput(); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath.hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } + std::visit(overloaded { + [&](TextHashMethod _) { + readFile(actualPath, caSink); + }, + [&](FileIngestionMethod m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + }, + }, outputHash.method); auto got = caSink.finish().first; HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); @@ -4098,13 +4107,10 @@ void DerivationGoal::registerOutputs() worker.store, { .name = outputPathName(drv->name, outputName), - .info = FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, - }, - rewriteRefs(), - }, + .info = contentAddressFromMethodHashAndRefs( + outputHash.method, + std::move(got), + rewriteRefs()), }, narHashAndSize.first, }; @@ -4132,13 +4138,14 @@ void DerivationGoal::registerOutputs() return newInfo0; }, [&](DerivationOutputCAFixed dof) { + auto wanted = getContentAddressHash(dof.ca); + auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, + .method = getContentAddressMethod(dof.ca), + .hashType = wanted.type, }); /* Check wanted hash */ - Hash & wanted = dof.hash.hash; assert(newInfo0.ca); auto got = getContentAddressHash(*newInfo0.ca); if (wanted != got) { @@ -4151,6 +4158,11 @@ void DerivationGoal::registerOutputs() wanted.to_string(SRI, true), got.to_string(SRI, true))); } + if (static_cast &>(newInfo0) != PathReferences {}) + delayedException = std::make_exception_ptr( + BuildError("illegal path references in fixed-output derivation '%s'", + worker.store.printStorePath(drvPath))); + return newInfo0; }, [&](DerivationOutputCAFloating dof) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index d68c60f4f..4226213b9 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -10,7 +10,7 @@ std::string FixedOutputHash::printMethodAlgo() const } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) assert(false); } +std::string makeContentAddressingPrefix(ContentAddressMethod m) { + return std::visit(overloaded { + [](TextHashMethod _) -> std::string { return "text:"; }, + [](FileIngestionMethod m2) { + /* Not prefixed for back compat with things that couldn't produce text before. */ + return makeFileIngestionPrefix(m2); + }, + }, m); +} + +ContentAddressMethod parseContentAddressingPrefix(std::string_view & m) +{ + ContentAddressMethod method = FileIngestionMethod::Flat; + if (splitPrefix(m, "r:")) + method = FileIngestionMethod::Recursive; + else if (splitPrefix(m, "text:")) + method = TextHashMethod {}; + return method; +} + + std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { return "fixed:" @@ -43,14 +64,14 @@ std::string renderContentAddress(ContentAddress ca) }, ca); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string renderContentAddressMethodAndHash(ContentAddressMethod cam, HashType ht) { return std::visit(overloaded { - [](TextHashMethod &th) { - return std::string{"text:"} + printHashType(htSHA256); + [&](TextHashMethod & th) { + return std::string{"text:"} + printHashType(ht); }, - [](FixedOutputHashMethod &fshm) { - return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); + [&](FileIngestionMethod & fim) { + return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); } }, cam); } @@ -58,7 +79,7 @@ std::string renderContentAddressMethod(ContentAddressMethod cam) /* Parses content address strings up to the hash. */ -static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -82,19 +103,19 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. HashType hashType = parseHashType_(); - if (hashType != htSHA256) - throw Error("text content address hash should use %s, but instead uses %s", - printHashType(htSHA256), printHashType(hashType)); - return TextHashMethod {}; + return { + TextHashMethod {}, + std::move(hashType), + }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; HashType hashType = parseHashType_(); - return FixedOutputHashMethod { - .fileIngestionMethod = method, - .hashType = std::move(hashType), + return { + std::move(method), + std::move(hashType), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); @@ -103,25 +124,24 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r ContentAddress parseContentAddress(std::string_view rawCa) { auto rest = rawCa; - ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit( - overloaded { - [&](TextHashMethod thm) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) - }); - }, - [&](FixedOutputHashMethod fohMethod) { - return ContentAddress(FixedOutputHash { - .method = fohMethod.fileIngestionMethod, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), - }); - }, - }, caMethod); + return std::visit(overloaded { + [&](TextHashMethod _) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, hashType) + }); + }, + [&](FileIngestionMethod fim) { + return ContentAddress(FixedOutputHash { + .method = fim, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }); + }, + }, caMethod); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +std::pair parseContentAddressMethod(std::string_view caMethod) { std::string_view asPrefix {std::string{caMethod} + ":"}; return parseContentAddressMethodPrefix(asPrefix); @@ -137,6 +157,42 @@ std::string renderContentAddress(std::optional ca) return ca ? renderContentAddress(*ca) : ""; } +ContentAddressWithReferences contentAddressFromMethodHashAndRefs( + ContentAddressMethod method, Hash && hash, PathReferences && refs) +{ + return std::visit(overloaded { + [&](TextHashMethod _) -> ContentAddressWithReferences { + if (refs.hasSelfReference) + throw UsageError("Cannot have a self reference with text hashing scheme"); + return TextInfo { + { .hash = std::move(hash) }, + std::move(refs.references), + }; + }, + [&](FileIngestionMethod m2) -> ContentAddressWithReferences { + return FixedOutputInfo { + { + .method = m2, + .hash = std::move(hash), + }, + std::move(refs), + }; + }, + }, method); +} + +ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences & ca) +{ + return std::visit(overloaded { + [](TextInfo th) -> ContentAddressMethod { + return TextHashMethod {}; + }, + [](FixedOutputInfo fsh) -> ContentAddressMethod { + return fsh.method; + }, + }, ca); +} + Hash getContentAddressHash(const ContentAddress & ca) { return std::visit(overloaded { @@ -160,4 +216,21 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { }, ca); } +Hash getContentAddressHash(const ContentAddressWithReferences & ca) +{ + return std::visit(overloaded { + [](TextInfo th) { + return th.hash; + }, + [](FixedOutputInfo fsh) { + return fsh.hash; + }, + }, ca); +} + +std::string printMethodAlgo(const ContentAddressWithReferences & ca) { + return makeContentAddressingPrefix(getContentAddressMethod(ca)) + + printHashType(getContentAddressHash(ca).type); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e15d76bd7..1fdd16e92 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -10,11 +10,45 @@ namespace nix { * Mini content address */ +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ +struct TextHashMethod : std::monostate { }; + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(FileIngestionMethod m); + + +/* Just the type of a content address. Combine with the hash itself, and we + have a `ContentAddress` as defined below. Combine that, in turn, with info + on references, and we have `ContentAddressWithReferences`, as defined + further below. */ +typedef std::variant< + TextHashMethod, + FileIngestionMethod +> ContentAddressMethod; + +/* Parse and pretty print the algorithm which indicates how the files + were ingested, with the the fixed output case not prefixed for back + compat. */ + +std::string makeContentAddressingPrefix(ContentAddressMethod m); + +ContentAddressMethod parseContentAddressingPrefix(std::string_view & m); + +/* Parse and pretty print a content addressing method and hash in a + nicer way, prefixing both cases. */ + +std::string renderContentAddressMethodAndHash(ContentAddressMethod cam, HashType ht); + +std::pair parseContentAddressMethod(std::string_view caMethod); + struct TextHash { Hash hash; @@ -27,6 +61,7 @@ struct FixedOutputHash { std::string printMethodAlgo() const; }; + /* We've accumulated several types of content-addressed paths over the years; fixed-output derivations support multiple hash algorithms and serialisation @@ -43,10 +78,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -57,24 +88,6 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; - -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); - -std::string renderContentAddressMethod(ContentAddressMethod caMethod); /* * References set @@ -92,6 +105,12 @@ struct PathReferences && hasSelfReference == other.hasSelfReference; } + bool operator != (const PathReferences & other) const + { + return references != other.references + || hasSelfReference != other.hasSelfReference; + } + /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; @@ -151,6 +170,14 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); +ContentAddressWithReferences contentAddressFromMethodHashAndRefs( + ContentAddressMethod method, Hash && hash, PathReferences && refs); + +ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences & ca); +Hash getContentAddressHash(const ContentAddressWithReferences & ca); + +std::string printMethodAlgo(const ContentAddressWithReferences &); + struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7ae88b49a..0e2ae4134 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -380,20 +380,23 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + auto [contentAddressMethod, hashType] = parseContentAddressMethod(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &_) { + [&](TextHashMethod _) { + if (hashType != htSHA256) + throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given", + printHashType(hashType)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod &fohm) { + [&](FileIngestionMethod fim) { if (!refs.empty()) throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + auto path = store->addToStoreFromDump(source, name, fim, hashType, repair); return store->queryPathInfo(path); }, }, contentAddressMethod); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 925a78083..e15f9e25a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" +#include "split.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" @@ -26,9 +27,10 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { - return store.makeFixedOutputPath( - outputPathName(drvName, outputName), - { hash, {} }); + return store.makeFixedOutputPathFromCA(StorePathDescriptor { + .name = outputPathName(drvName, outputName), + .info = ca, + }); } @@ -150,23 +152,19 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hash) + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) { if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (string(hashAlgo, 0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } + ContentAddressMethod method = parseContentAddressingPrefix(hashAlgo); const auto hashType = parseHashType(hashAlgo); - if (hash != "") { + if (hashS != "") { validatePath(pathS); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput { .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, + // FIXME non-trivial fixed refs set + .ca = contentAddressFromMethodHashAndRefs( + method, std::move(hash), {}), }, }; } else { @@ -317,12 +315,12 @@ string Derivation::unparse(const Store & store, bool maskOutputs, }, [&](DerivationOutputCAFixed dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); - s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, printMethodAlgo(dof.ca)); + s += ','; printUnquotedString(s, getContentAddressHash(dof.ca).to_string(Base16, false)); }, [&](DerivationOutputCAFloating dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, }, i.second.output); @@ -482,8 +480,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.output); auto hash = hashString(htSHA256, "fixed:out:" - + dof.hash.printMethodAlgo() + ":" - + dof.hash.hash.to_string(Base16, false) + ":" + + printMethodAlgo(dof.ca) + ":" + + getContentAddressHash(dof.ca).to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -612,12 +610,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](DerivationOutputCAFixed dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) - << dof.hash.printMethodAlgo() - << dof.hash.hash.to_string(Base16, false); + << printMethodAlgo(dof.ca) + << getContentAddressHash(dof.ca).to_string(Base16, false); }, [&](DerivationOutputCAFloating dof) { out << "" - << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << (makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, }, i.second.output); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e2e5578a8..3bfaabad3 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -27,7 +27,7 @@ struct DerivationOutputInputAddressed according to that fixed output. */ struct DerivationOutputCAFixed { - FixedOutputHash hash; /* hash used for expected hash computation */ + ContentAddressWithReferences ca; /* hash and refs used for validating output */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; @@ -37,7 +37,7 @@ struct DerivationOutputCAFixed struct DerivationOutputCAFloating { /* information used for expected hash computation */ - FileIngestionMethod method; + ContentAddressMethod method; HashType hashType; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e6b02cce6..f573845f1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); + auto path = dof.path(*this, drvName, i.first); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 052936a8b..71fa81546 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -111,9 +111,21 @@ void Store::computeFSClosure(const StorePath & startPath, std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) { - if (auto v = std::get_if(&out->second.output)) - return v->hash; + if (out == drv.outputs.end()) + return std::nullopt; + if (auto dof = std::get_if(&out->second.output)) { + return std::visit(overloaded { + [&](TextInfo ti) -> std::optional { + if (!ti.references.empty()) + return std::nullopt; + return static_cast(ti); + }, + [&](FixedOutputInfo fi) -> std::optional { + if (fi.references != PathReferences {}) + return std::nullopt; + return static_cast(fi); + }, + }, dof->ca); } return std::nullopt; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5ff787ed2..4f71d8120 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -457,6 +457,7 @@ ref RemoteStore::addCAToStore( Source & dump, const string & name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair) { @@ -468,7 +469,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << renderContentAddressMethodAndHash(caMethod, hashType); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -484,18 +485,21 @@ ref RemoteStore::addCAToStore( std::visit(overloaded { [&](TextHashMethod thm) -> void { + if (hashType != htSHA256) + throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given", + printHashType(hashType)); std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; worker_proto::write(*this, conn->to, references); conn.processStderr(); }, - [&](FixedOutputHashMethod fohm) -> void { + [&](FileIngestionMethod fim) -> void { conn->to << wopAddToStore << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); + << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashType); try { conn->to.written = 0; @@ -503,7 +507,7 @@ ref RemoteStore::addCAToStore( connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::Recursive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -536,7 +540,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; + return addCAToStore(dump, name, method, hashType, references, repair)->path; } @@ -597,7 +601,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; + return addCAToStore(source, name, TextHashMethod {}, htSHA256, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f78fcb02..0260e3221 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -68,6 +68,7 @@ public: Source & dump, const string & name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2542537d3..0799f21eb 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -76,11 +76,12 @@ struct CmdShowDerivation : InstallablesCommand }, [&](DerivationOutputCAFixed dof) { outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); - outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); - outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); + outputObj.attr("hashAlgo", printMethodAlgo(dof.ca)); + outputObj.attr("hash", getContentAddressHash(dof.ca).to_string(Base16, false)); + // FIXME print refs? }, [&](DerivationOutputCAFloating dof) { - outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + outputObj.attr("hashAlgo", makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType)); }, }, output.output); } diff --git a/tests/local.mk b/tests/local.mk index a1929f96d..ce50a4ca5 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -35,7 +35,8 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ - content-addressed.sh + content-addressed.sh \ + text-hashed-output.sh # parallel.sh # build-remote-content-addressed-fixed.sh \ diff --git a/tests/text-hashed-output.nix b/tests/text-hashed-output.nix new file mode 100644 index 000000000..23434c0a1 --- /dev/null +++ b/tests/text-hashed-output.nix @@ -0,0 +1,29 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + root = mkDerivation { + name = "text-hashed-root"; + buildCommand = '' + set -x + echo "Building a CA derivation" + mkdir -p $out + echo "Hello World" > $out/hello + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + dependent = mkDerivation { + name = "text-hashed-root.drv"; + buildCommand = '' + echo "Copying the derivation" + cp ${root.drvPath} $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }; +} diff --git a/tests/text-hashed-output.sh b/tests/text-hashed-output.sh new file mode 100644 index 000000000..2ee3d6590 --- /dev/null +++ b/tests/text-hashed-output.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named root, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of root, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate the root derivation +# - build the dependent derivation +# - check that the path of the output coincides with that of the original derivation + +drv=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A root) +nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" + +drvDep=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A dependent) +nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drvDep" + +out1=$(nix-build --experimental-features ca-derivations ./text-hashed-output.nix -A dependent --no-out-link) + +nix --experimental-features 'nix-command ca-derivations' path-info $drv --derivation --json | jq +nix --experimental-features 'nix-command ca-derivations' path-info $out1 --derivation --json | jq + +test $out1 == $drv