Merge pull request #3959 from obsidiansystems/ca-drv-exotic

Derivations can output "text-hashed" data
This commit is contained in:
John Ericson 2023-05-10 10:41:59 -04:00 committed by GitHub
commit 53a1354acf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 572 additions and 175 deletions

1
.gitignore vendored
View file

@ -85,6 +85,7 @@ perl/Makefile.config
/tests/shell.drv /tests/shell.drv
/tests/config.nix /tests/config.nix
/tests/ca/config.nix /tests/ca/config.nix
/tests/dyn-drv/config.nix
/tests/repl-result-out /tests/repl-result-out
# /tests/lang/ # /tests/lang/

View file

@ -1100,7 +1100,7 @@ drvName, Bindings * attrs, Value & v)
bool isImpure = false; bool isImpure = false;
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::string outputHashAlgo;
std::optional<FileIngestionMethod> ingestionMethod; std::optional<ContentAddressMethod> ingestionMethod;
StringSet outputs; StringSet outputs;
outputs.insert("out"); outputs.insert("out");
@ -1113,7 +1113,10 @@ drvName, Bindings * attrs, Value & v)
auto handleHashMode = [&](const std::string_view s) { auto handleHashMode = [&](const std::string_view s) {
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else else if (s == "text") {
experimentalFeatureSettings.require(Xp::DynamicDerivations);
ingestionMethod = TextIngestionMethod {};
} else
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
@ -1280,11 +1283,16 @@ drvName, Bindings * attrs, Value & v)
})); }));
/* Check whether the derivation name is valid. */ /* Check whether the derivation name is valid. */
if (isDerivation(drvName)) if (isDerivation(drvName) &&
!(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
outputs.size() == 1 &&
*(outputs.begin()) == "out"))
{
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
})); }));
}
if (outputHash) { if (outputHash) {
/* Handle fixed-output derivations. /* Handle fixed-output derivations.
@ -1300,21 +1308,15 @@ drvName, Bindings * attrs, Value & v)
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
.hash = { DerivationOutput::CAFixed dof {
.method = method, .ca = ContentAddress::fromParts(
.hash = h, std::move(method),
}, std::move(h)),
.references = {}, };
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
drv.outputs.insert_or_assign("out", drv.outputs.insert_or_assign("out", std::move(dof));
DerivationOutput::CAFixed {
.hash = FixedOutputHash {
.method = method,
.hash = std::move(h),
},
});
} }
else if (contentAddressed || isImpure) { else if (contentAddressed || isImpure) {
@ -1332,13 +1334,13 @@ drvName, Bindings * attrs, Value & v)
if (isImpure) if (isImpure)
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput::Impure { DerivationOutput::Impure {
.method = method, .method = method.raw,
.hashType = ht, .hashType = ht,
}); });
else else
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating { DerivationOutput::CAFloating {
.method = method, .method = method.raw,
.hashType = ht, .hashType = ht,
}); });
} }

View file

@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
) )
) )
); );
else else {
auto * cap = getDerivationCA(*drv);
addWaitee(upcast_goal(worker.makePathSubstitutionGoal( addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path, status.known->path,
buildMode == bmRepair ? Repair : NoRepair, buildMode == bmRepair ? Repair : NoRepair,
getDerivationCA(*drv)))); cap ? std::optional { *cap } : std::nullopt)));
}
} }
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */

View file

@ -2426,37 +2426,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
throw BuildError( throw BuildError(
"output path %1% without valid stats info", "output path %1% without valid stats info",
actualPath); actualPath);
if (outputHash.method == FileIngestionMethod::Flat) { if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
{
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
"output path '%1%' should be a non-executable regular file " "output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)", "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
actualPath); actualPath);
} }
rewriteOutput(); rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */ /* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() }; std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart }; HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) { std::visit(overloaded {
case FileIngestionMethod::Recursive: [&](const TextIngestionMethod &) {
dumpPath(actualPath, caSink); readFile(actualPath, caSink);
break; },
case FileIngestionMethod::Flat: [&](const FileIngestionMethod & m2) {
readFile(actualPath, caSink); switch (m2) {
break; case FileIngestionMethod::Recursive:
} dumpPath(actualPath, caSink);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
},
}, outputHash.method.raw);
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto optCA = ContentAddressWithReferences::fromPartsOpt(
outputHash.method,
std::move(got),
rewriteRefs());
if (!optCA) {
// TODO track distinct failure modes separately (at the time of
// writing there is just one but `nullopt` is unclear) so this
// message can't get out of sync.
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
}
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store, worker.store,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
FixedOutputInfo { *std::move(optCA),
.hash = {
.method = outputHash.method,
.hash = got,
},
.references = rewriteRefs(),
},
Hash::dummy, Hash::dummy,
}; };
if (*scratchPath != newInfo0.path) { if (*scratchPath != newInfo0.path) {
@ -2503,13 +2517,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
auto wanted = dof.ca.getHash();
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.hash.method, .method = dof.ca.getMethod(),
.hashType = dof.hash.hash.type, .hashType = wanted.type,
}); });
/* Check wanted hash */ /* Check wanted hash */
const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = newInfo0.ca->getHash(); auto got = newInfo0.ca->getHash();
if (wanted != got) { if (wanted != got) {
@ -2522,6 +2537,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
wanted.to_string(SRI, true), wanted.to_string(SRI, true),
got.to_string(SRI, true))); got.to_string(SRI, true)));
} }
if (!newInfo0.references.empty())
delayedException = std::make_exception_ptr(
BuildError("illegal path references in fixed-output derivation '%s'",
worker.store.printStorePath(drvPath)));
return newInfo0; return newInfo0;
}, },

View file

@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
} }
} }
std::string ContentAddressMethod::renderPrefix() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string { return "text:"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2);
},
}, raw);
}
ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
{
ContentAddressMethod method = FileIngestionMethod::Flat;
if (splitPrefix(m, "r:"))
method = FileIngestionMethod::Recursive;
else if (splitPrefix(m, "text:"))
method = TextIngestionMethod {};
return method;
}
std::string ContentAddress::render() const std::string ContentAddress::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -36,14 +57,14 @@ std::string ContentAddress::render() const
}, raw); }, raw);
} }
std::string ContentAddressMethod::render() const std::string ContentAddressMethod::render(HashType ht) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextHashMethod & th) { [&](const TextIngestionMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(ht);
}, },
[](const FixedOutputHashMethod & fshm) { [&](const FileIngestionMethod & fim) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
} }
}, raw); }, raw);
} }
@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const
/** /**
* Parses content address strings up to the hash. * Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
{ {
std::string_view wholeInput { rest }; std::string_view wholeInput { rest };
@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
if (prefix == "text") { if (prefix == "text") {
// No parsing of the ingestion method, "text" only support flat. // No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
if (hashType != htSHA256) return {
throw Error("text content address hash should use %s, but instead uses %s", TextIngestionMethod {},
printHashType(htSHA256), printHashType(hashType)); std::move(hashType),
return TextHashMethod {}; };
} else if (prefix == "fixed") { } else if (prefix == "fixed") {
// Parse method // Parse method
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:")) if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
return FixedOutputHashMethod { return {
.fileIngestionMethod = method, std::move(method),
.hashType = std::move(hashType), std::move(hashType),
}; };
} else } else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
} }
ContentAddress ContentAddress::parse(std::string_view rawCa) { ContentAddress ContentAddress::parse(std::string_view rawCa)
{
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
auto hashType = hashType_; // work around clang bug
return std::visit( return std::visit(overloaded {
overloaded { [&](TextIngestionMethod &) {
[&](TextHashMethod & thm) { return ContentAddress(TextHash {
return ContentAddress(TextHash { .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) });
}); },
}, [&](FileIngestionMethod & fim) {
[&](FixedOutputHashMethod & fohMethod) { return ContentAddress(FixedOutputHash {
return ContentAddress(FixedOutputHash { .method = fim,
.method = fohMethod.fileIngestionMethod, .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), });
}); },
}, }, caMethod.raw);
}, caMethod.raw);
} }
ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
{ {
std::string asPrefix = std::string{caMethod} + ":"; std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference // parseContentAddressMethodPrefix takes its argument by reference
@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
return ca ? ca->render() : ""; return ca ? ca->render() : "";
} }
ContentAddress ContentAddress::fromParts(
ContentAddressMethod method, Hash hash) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> ContentAddress {
return TextHash {
.hash = std::move(hash),
};
},
[&](FileIngestionMethod m2) -> ContentAddress {
return FixedOutputHash {
.method = std::move(m2),
.hash = std::move(hash),
};
},
}, method.raw);
}
ContentAddressMethod ContentAddress::getMethod() const
{
return std::visit(overloaded {
[](const TextHash & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputHash & fsh) -> ContentAddressMethod {
return fsh.method;
},
}, raw);
}
const Hash & ContentAddress::getHash() const const Hash & ContentAddress::getHash() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const
}, raw); }, raw);
} }
std::string ContentAddress::printMethodAlgo() const
{
return getMethod().renderPrefix()
+ printHashType(getHash().type);
}
bool StoreReferences::empty() const bool StoreReferences::empty() const
{ {
return !self && others.empty(); return !self && others.empty();
@ -156,7 +214,8 @@ size_t StoreReferences::size() const
return (self ? 1 : 0) + others.size(); return (self ? 1 : 0) + others.size();
} }
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
{
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences { [&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo { return TextInfo {
@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
}, ca.raw); }, ca.raw);
} }
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
if (refs.self)
return std::nullopt;
return ContentAddressWithReferences {
TextInfo {
.hash = { .hash = std::move(hash) },
.references = std::move(refs.others),
}
};
},
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
return ContentAddressWithReferences {
FixedOutputInfo {
.hash = {
.method = m2,
.hash = std::move(hash),
},
.references = std::move(refs),
}
};
},
}, method.raw);
}
ContentAddressMethod ContentAddressWithReferences::getMethod() const
{
return std::visit(overloaded {
[](const TextInfo & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputInfo & fsh) -> ContentAddressMethod {
return fsh.hash.method;
},
}, raw);
}
Hash ContentAddressWithReferences::getHash() const
{
return std::visit(overloaded {
[](const TextInfo & th) {
return th.hash.hash;
},
[](const FixedOutputInfo & fsh) {
return fsh.hash.hash;
},
}, raw);
}
} }

View file

@ -21,8 +21,14 @@ namespace nix {
* *
* Somewhat obscure, used by \ref Derivation derivations and * Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently. * `builtins.toFile` currently.
*
* TextIngestionMethod is identical to FileIngestionMethod::Fixed except that
* the former may not have self-references and is tagged `text:${algo}:${hash}`
* rather than `fixed:${algo}:${hash}`. The contents of the store path are
* ingested and hashed identically, aside from the slightly different tag and
* restriction on self-references.
*/ */
struct TextHashMethod : std::monostate { }; struct TextIngestionMethod : std::monostate { };
/** /**
* An enumeration of the main ways we can serialize file system * An enumeration of the main ways we can serialize file system
@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t {
*/ */
std::string makeFileIngestionPrefix(FileIngestionMethod m); std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
};
/** /**
* An enumeration of all the ways we can serialize file system objects. * An enumeration of all the ways we can serialize file system objects.
* *
@ -64,8 +63,8 @@ struct FixedOutputHashMethod {
struct ContentAddressMethod struct ContentAddressMethod
{ {
typedef std::variant< typedef std::variant<
TextHashMethod, TextIngestionMethod,
FixedOutputHashMethod FileIngestionMethod
> Raw; > Raw;
Raw raw; Raw raw;
@ -77,9 +76,36 @@ struct ContentAddressMethod
: raw(std::forward<decltype(arg)>(arg)...) : raw(std::forward<decltype(arg)>(arg)...)
{ } { }
static ContentAddressMethod parse(std::string_view rawCaMethod);
std::string render() const; /**
* Parse the prefix tag which indicates how the files
* were ingested, with the fixed output case not prefixed for back
* compat.
*
* @param [in] m A string that should begin with the prefix.
* @param [out] m The remainder of the string after the prefix.
*/
static ContentAddressMethod parsePrefix(std::string_view & m);
/**
* Render the prefix tag which indicates how the files wre ingested.
*
* The rough inverse of `parsePrefix()`.
*/
std::string renderPrefix() const;
/**
* Parse a content addressing method and hash type.
*/
static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
/**
* Render a content addressing method and hash type in a
* nicer way, prefixing both cases.
*
* The rough inverse of `parse()`.
*/
std::string render(HashType ht) const;
}; };
@ -147,8 +173,9 @@ struct ContentAddress
{ } { }
/** /**
* Compute the content-addressability assertion (ValidPathInfo::ca) for * Compute the content-addressability assertion
* paths created by Store::makeFixedOutputPath() / Store::addToStore(). * (`ValidPathInfo::ca`) for paths created by
* `Store::makeFixedOutputPath()` / `Store::addToStore()`.
*/ */
std::string render() const; std::string render() const;
@ -156,9 +183,27 @@ struct ContentAddress
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt); static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
/**
* Create a `ContentAddress` from 2 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*/
static ContentAddress fromParts(
ContentAddressMethod method, Hash hash) noexcept;
ContentAddressMethod getMethod() const;
const Hash & getHash() const; const Hash & getHash() const;
std::string printMethodAlgo() const;
}; };
/**
* Render the `ContentAddress` if it exists to a string, return empty
* string otherwise.
*/
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -244,10 +289,29 @@ struct ContentAddressWithReferences
{ } { }
/** /**
* Create a ContentAddressWithReferences from a mere ContentAddress, by * Create a `ContentAddressWithReferences` from a mere
* assuming no references in all cases. * `ContentAddress`, by claiming no references.
*/ */
static ContentAddressWithReferences withoutRefs(const ContentAddress &); static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept;
/**
* Create a `ContentAddressWithReferences` from 3 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*
* @param refs References to other store objects or oneself.
*
* Do note that not all combinations are supported; `nullopt` is
* returns for invalid combinations.
*/
static std::optional<ContentAddressWithReferences> fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
ContentAddressMethod getMethod() const;
Hash getHash() const;
}; };
} }

View file

@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
auto hashType = hashType_; // work around clang bug
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHashMethod &) { [&](const TextIngestionMethod &) {
if (hashType != htSHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashType(hashType));
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](const FixedOutputHashMethod & fohm) { [&](const FileIngestionMethod & fim) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod.raw); }, contentAddressMethod.raw);

View file

@ -2,6 +2,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include "split.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
@ -35,9 +36,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{ {
return store.makeFixedOutputPath( return store.makeFixedOutputPathFromCA(
outputPathName(drvName, outputName), outputPathName(drvName, outputName),
{ hash, {} }); ContentAddressWithReferences::withoutRefs(ca));
} }
@ -211,29 +212,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, static DerivationOutput parseDerivationOutput(const Store & store,
std::string_view pathS, std::string_view hashAlgo, std::string_view hash) std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
{ {
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
if (hashAlgo.substr(0, 2) == "r:") { if (method == TextIngestionMethod {})
method = FileIngestionMethod::Recursive; experimentalFeatureSettings.require(Xp::DynamicDerivations);
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash == "impure") { if (hashS == "impure") {
experimentalFeatureSettings.require(Xp::ImpureDerivations); experimentalFeatureSettings.require(Xp::ImpureDerivations);
assert(pathS == ""); assert(pathS == "");
return DerivationOutput::Impure { return DerivationOutput::Impure {
.method = std::move(method), .method = std::move(method),
.hashType = std::move(hashType), .hashType = std::move(hashType),
}; };
} else if (hash != "") { } else if (hashS != "") {
validatePath(pathS); validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
return DerivationOutput::CAFixed { return DerivationOutput::CAFixed {
.hash = FixedOutputHash { .ca = ContentAddress::fromParts(
.method = std::move(method), std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), std::move(hash)),
},
}; };
} else { } else {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
@ -393,12 +392,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
@ -409,7 +408,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutputImpure & doi) { [&](const DerivationOutputImpure & doi) {
// FIXME // FIXME
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure"); s += ','; printUnquotedString(s, "impure");
} }
}, i.second.raw()); }, i.second.raw());
@ -626,8 +625,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw()); auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":" + dof.ca.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":" + dof.ca.getHash().to_string(Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
@ -777,12 +776,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first)) out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo() << dof.ca.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false); << dof.ca.getHash().to_string(Base16, false);
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
out << "" out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << (dof.method.renderPrefix() + printHashType(dof.hashType))
<< ""; << "";
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
@ -792,7 +791,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}, },
[&](const DerivationOutput::Impure & doi) { [&](const DerivationOutput::Impure & doi) {
out << "" out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) << (doi.method.renderPrefix() + printHashType(doi.hashType))
<< "impure"; << "impure";
}, },
}, i.second.raw()); }, i.second.raw());
@ -942,7 +941,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} }); auto path = dof.path(store, drvName, i.first);
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {
@ -971,15 +970,16 @@ nlohmann::json DerivationOutput::toJSON(
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.hash.printMethodAlgo(); res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.hash.hash.to_string(Base16, false); res["hash"] = dof.ca.getHash().to_string(Base16, false);
// FIXME print refs?
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
}, },
[&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) { [&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
res["impure"] = true; res["impure"] = true;
}, },
}, raw()); }, raw());
@ -998,15 +998,15 @@ DerivationOutput DerivationOutput::fromJSON(
for (const auto & [key, _] : json) for (const auto & [key, _] : json)
keys.insert(key); keys.insert(key);
auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> { auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
std::string hashAlgo = json["hashAlgo"]; std::string hashAlgo = json["hashAlgo"];
auto method = FileIngestionMethod::Flat; // remaining to parse, will be mutated by parsers
if (hashAlgo.substr(0, 2) == "r:") { std::string_view s = hashAlgo;
method = FileIngestionMethod::Recursive; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
hashAlgo = hashAlgo.substr(2); if (method == TextIngestionMethod {})
} xpSettings.require(Xp::DynamicDerivations);
auto hashType = parseHashType(hashAlgo); auto hashType = parseHashType(s);
return { method, hashType }; return { std::move(method), std::move(hashType) };
}; };
if (keys == (std::set<std::string_view> { "path" })) { if (keys == (std::set<std::string_view> { "path" })) {
@ -1018,10 +1018,9 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) { else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
auto dof = DerivationOutput::CAFixed { auto dof = DerivationOutput::CAFixed {
.hash = { .ca = ContentAddress::fromParts(
.method = method, std::move(method),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
},
}; };
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
throw Error("Path doesn't match derivation output"); throw Error("Path doesn't match derivation output");
@ -1032,8 +1031,8 @@ DerivationOutput DerivationOutput::fromJSON(
xpSettings.require(Xp::CaDerivations); xpSettings.require(Xp::CaDerivations);
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
return DerivationOutput::CAFloating { return DerivationOutput::CAFloating {
.method = method, .method = std::move(method),
.hashType = hashType, .hashType = std::move(hashType),
}; };
} }
@ -1045,7 +1044,7 @@ DerivationOutput DerivationOutput::fromJSON(
xpSettings.require(Xp::ImpureDerivations); xpSettings.require(Xp::ImpureDerivations);
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
return DerivationOutput::Impure { return DerivationOutput::Impure {
.method = method, .method = std::move(method),
.hashType = hashType, .hashType = hashType,
}; };
} }

View file

@ -36,9 +36,11 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed struct DerivationOutputCAFixed
{ {
/** /**
* hash used for expected hash computation * Method and hash used for expected hash computation.
*
* References are not allowed by fiat.
*/ */
FixedOutputHash hash; ContentAddress ca;
/** /**
* Return the \ref StorePath "store path" corresponding to this output * Return the \ref StorePath "store path" corresponding to this output
@ -48,7 +50,7 @@ struct DerivationOutputCAFixed
*/ */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
GENERATE_CMP(DerivationOutputCAFixed, me->hash); GENERATE_CMP(DerivationOutputCAFixed, me->ca);
}; };
/** /**
@ -61,7 +63,7 @@ struct DerivationOutputCAFloating
/** /**
* How the file system objects will be serialized for hashing * How the file system objects will be serialized for hashing
*/ */
FileIngestionMethod method; ContentAddressMethod method;
/** /**
* How the serialization will be hashed * How the serialization will be hashed
@ -88,7 +90,7 @@ struct DerivationOutputImpure
/** /**
* How the file system objects will be serialized for hashing * How the file system objects will be serialized for hashing
*/ */
FileIngestionMethod method; ContentAddressMethod method;
/** /**
* How the serialization will be hashed * How the serialization will be hashed
@ -343,12 +345,14 @@ struct Derivation : BasicDerivation
Store & store, Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
/* Check that the derivation is valid and does not present any /**
illegal states. * Check that the derivation is valid and does not present any
* illegal states.
This is mainly a matter of checking the outputs, where our C++ *
representation supports all sorts of combinations we do not yet * This is mainly a matter of checking the outputs, where our C++
allow. */ * representation supports all sorts of combinations we do not yet
* allow.
*/
void checkInvariants(Store & store, const StorePath & drvPath) const; void checkInvariants(Store & store, const StorePath & drvPath) const;
Derivation() = default; Derivation() = default;

View file

@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath,
} }
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) const ContentAddress * getDerivationCA(const BasicDerivation & drv)
{ {
auto out = drv.outputs.find("out"); auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) { if (out == drv.outputs.end())
if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw())) return nullptr;
return v->hash; if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
return &dof->ca;
} }
return std::nullopt; return nullptr;
} }
void Store::queryMissing(const std::vector<DerivedPath> & targets, void Store::queryMissing(const std::vector<DerivedPath> & targets,
@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
if (drvState_->lock()->done) return; if (drvState_->lock()->done) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); auto * cap = getDerivationCA(*drv);
querySubstitutablePathInfos({
{
outPath,
cap ? std::optional { *cap } : std::nullopt,
},
}, infos);
if (infos.empty()) { if (infos.empty()) {
drvState_->lock()->done = true; drvState_->lock()->done = true;

View file

@ -597,6 +597,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
Source & dump, Source & dump,
std::string_view name, std::string_view name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair) RepairFlag repair)
{ {
@ -608,7 +609,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< caMethod.render(); << caMethod.render(hashType);
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -628,26 +629,29 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
std::visit(overloaded { std::visit(overloaded {
[&](const TextHashMethod & thm) -> void { [&](const TextIngestionMethod & thm) -> void {
if (hashType != htSHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashType(hashType));
std::string s = dump.drain(); std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s; conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn.processStderr(); conn.processStderr();
}, },
[&](const FixedOutputHashMethod & fohm) -> void { [&](const FileIngestionMethod & fim) -> void {
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) << (fim == FileIngestionMethod::Recursive ? 1 : 0)
<< printHashType(fohm.hashType); << printHashType(hashType);
try { try {
conn->to.written = 0; conn->to.written = 0;
connections->incCapacity(); connections->incCapacity();
{ {
Finally cleanup([&]() { connections->decCapacity(); }); Finally cleanup([&]() { connections->decCapacity(); });
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { if (fim == FileIngestionMethod::Recursive) {
dump.drainInto(conn->to); dump.drainInto(conn->to);
} else { } else {
std::string contents = dump.drain(); std::string contents = dump.drain();
@ -678,7 +682,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
{ {
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; return addCAToStore(dump, name, method, hashType, references, repair)->path;
} }
@ -778,7 +782,7 @@ StorePath RemoteStore::addTextToStore(
RepairFlag repair) RepairFlag repair)
{ {
StringSource source(s); StringSource source(s);
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
} }
void RemoteStore::registerDrvOutput(const Realisation & info) void RemoteStore::registerDrvOutput(const Realisation & info)

View file

@ -78,6 +78,7 @@ public:
Source & dump, Source & dump,
std::string_view name, std::string_view name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair); RepairFlag repair);

View file

@ -1022,7 +1022,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
*/ */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences( std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store, Store & store,

View file

@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest
} }
}; };
class DynDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
}
};
class ImpureDerivationTest : public DerivationTest class ImpureDerivationTest : public DerivationTest
{ {
void SetUp() override void SetUp() override
@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed,
}), }),
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DerivationTest, caFixed, TEST_JSON(DerivationTest, caFixedFlat,
R"({
"hashAlgo": "sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),
"drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedNAR,
R"({ R"({
"hashAlgo": "r:sha256", "hashAlgo": "r:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})", })",
(DerivationOutput::CAFixed { (DerivationOutput::CAFixed {
.hash = { .ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive, .method = FileIngestionMethod::Recursive,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
}, },
}), }),
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DynDerivationTest, caFixedText,
R"({
"hashAlgo": "text:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = TextHash {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),
"drv-name", "output-name")
TEST_JSON(CaDerivationTest, caFloating, TEST_JSON(CaDerivationTest, caFloating,
R"({ R"({
"hashAlgo": "r:sha256" "hashAlgo": "r:sha256"

View file

@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
std::string_view description; std::string_view description;
}; };
constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
{ {
.tag = Xp::CaDerivations, .tag = Xp::CaDerivations,
.name = "ca-derivations", .name = "ca-derivations",
@ -199,6 +199,16 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
networking. networking.
)", )",
}, },
{
.tag = Xp::DynamicDerivations,
.name = "dynamic-derivations",
.description = R"(
Allow the use of a few things related to dynamic derivations:
- "text hashing" derivation outputs, so we can build .drv
files.
)",
},
}}; }};
static_assert( static_assert(

View file

@ -29,6 +29,7 @@ enum struct ExperimentalFeature
Cgroups, Cgroups,
DiscardReferences, DiscardReferences,
DaemonTrustOverride, DaemonTrustOverride,
DynamicDerivations,
}; };
/** /**

8
tests/dyn-drv/common.sh Normal file
View file

@ -0,0 +1,8 @@
source ../common.sh
# Need backend to support text-hashing too
requireDaemonNewerThan "2.16.0pre20230419"
enableFeatures "ca-derivations dynamic-derivations"
restartDaemon

1
tests/dyn-drv/config.nix.in Symbolic link
View file

@ -0,0 +1 @@
../config.nix.in

View file

@ -0,0 +1,33 @@
with import ./config.nix;
let innerName = "foo"; in
mkDerivation rec {
name = "${innerName}.drv";
SHELL = shell;
requiredSystemFeatures = [ "recursive-nix" ];
drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath;
buildCommand = ''
export NIX_CONFIG='experimental-features = nix-command ca-derivations'
PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH
# JSON of pre-existing drv
nix derivation show $drv | jq .[] > drv0.json
# Fix name
jq < drv0.json '.name = "${innerName}"' > drv1.json
# Extend `buildCommand`
jq < drv1.json '.env.buildCommand += "echo \"I am alive!\" >> $out/hello\n"' > drv0.json
# Used as our output
cp $(nix derivation add < drv0.json) $out
'';
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
}

View file

@ -0,0 +1,25 @@
source common.sh
# FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
enableFeatures 'recursive-nix'
restartDaemon
clearStore
rm -f $TEST_ROOT/result
EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq))
export EXTRA_PATH
# Will produce a drv
metaDrv=$(nix-instantiate ./recursive-mod-json.nix)
# computed "dynamic" derivation
drv=$(nix-store -r $metaDrv)
# build that dyn drv
res=$(nix-store -r $drv)
grep 'I am alive!' $res/hello

View file

@ -0,0 +1,29 @@
with import ./config.nix;
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
rec {
hello = mkDerivation {
name = "hello";
buildCommand = ''
set -x
echo "Building a CA derivation"
mkdir -p $out
echo "Hello World" > $out/hello
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
producingDrv = mkDerivation {
name = "hello.drv";
buildCommand = ''
echo "Copying the derivation"
cp ${builtins.unsafeDiscardOutputDependency hello.drvPath} $out
'';
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
};
}

View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
source common.sh
# In the corresponding nix file, we have two derivations: the first, named root,
# is a normal recursive derivation, while the second, named dependent, has the
# new outputHashMode "text". Note that in "dependent", we don't refer to the
# build output of root, but only to the path of the drv file. For this reason,
# we only need to:
#
# - instantiate the root derivation
# - build the dependent derivation
# - check that the path of the output coincides with that of the original derivation
drv=$(nix-instantiate ./text-hashed-output.nix -A hello)
nix show-derivation "$drv"
drvProducingDrv=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
nix show-derivation "$drvProducingDrv"
out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link)
nix path-info $drv --derivation --json | jq
nix path-info $out1 --derivation --json | jq
test $out1 == $drv

View file

@ -111,6 +111,8 @@ nix_tests = \
ca/derivation-json.sh \ ca/derivation-json.sh \
import-derivation.sh \ import-derivation.sh \
ca/import-derivation.sh \ ca/import-derivation.sh \
dyn-drv/text-hashed-output.sh \
dyn-drv/recursive-mod-json.sh \
nix_path.sh \ nix_path.sh \
case-hack.sh \ case-hack.sh \
placeholders.sh \ placeholders.sh \
@ -138,11 +140,19 @@ ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh nix_tests += compute-levels.sh
endif endif
install-tests += $(foreach x, $(nix_tests), tests/$(x)) install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
clean-files += $(d)/common/vars-and-functions.sh $(d)/config.nix $(d)/ca/config.nix clean-files += \
$(d)/common/vars-and-functions.sh \
$(d)/config.nix \
$(d)/ca/config.nix \
$(d)/dyn-drv/config.nix
test-deps += tests/common/vars-and-functions.sh tests/config.nix tests/ca/config.nix test-deps += \
tests/common/vars-and-functions.sh \
tests/config.nix \
tests/ca/config.nix \
tests/dyn-drv/config.nix
ifeq ($(BUILD_SHARED_LIBS), 1) ifeq ($(BUILD_SHARED_LIBS), 1)
test-deps += tests/plugins/libplugintest.$(SO_EXT) test-deps += tests/plugins/libplugintest.$(SO_EXT)

View file

@ -1,11 +1,11 @@
source common.sh source common.sh
sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf
restartDaemon
# FIXME # FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
enableFeatures 'recursive-nix'
restartDaemon
clearStore clearStore
rm -f $TEST_ROOT/result rm -f $TEST_ROOT/result