TextHashMethod -> TextIngestionMethod, gate with XP feature

I suppose we can use `dynamic-derivations` for the few things we neeed.
This commit is contained in:
John Ericson 2023-04-17 19:02:45 -04:00
parent f56c4a5bdf
commit 668377f217
11 changed files with 53 additions and 29 deletions

View file

@ -1105,8 +1105,10 @@ drvName, Bindings * attrs, Value & v)
auto handleHashMode = [&](const std::string_view s) { auto handleHashMode = [&](const std::string_view s) {
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else if (s == "text") ingestionMethod = TextHashMethod {}; else if (s == "text") {
else experimentalFeatureSettings.require(Xp::DynamicDerivations);
ingestionMethod = TextIngestionMethod {};
} else
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
@ -1274,11 +1276,16 @@ drvName, Bindings * attrs, Value & v)
})); }));
/* Check whether the derivation name is valid. */ /* Check whether the derivation name is valid. */
if (isDerivation(drvName) && ingestionMethod != ContentAddressMethod { TextHashMethod { } }) if (isDerivation(drvName) &&
!(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
outputs.size() == 1 &&
*(outputs.begin()) == "out"))
{
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
})); }));
}
if (outputHash) { if (outputHash) {
/* Handle fixed-output derivations. /* Handle fixed-output derivations.

View file

@ -2427,7 +2427,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
"output path %1% without valid stats info", "output path %1% without valid stats info",
actualPath); actualPath);
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextHashMethod {} }) outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
{ {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@ -2441,7 +2441,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
std::string oldHashPart { scratchPath->hashPart() }; std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart }; HashModuloSink caSink { outputHash.hashType, oldHashPart };
std::visit(overloaded { std::visit(overloaded {
[&](const TextHashMethod &) { [&](const TextIngestionMethod &) {
readFile(actualPath, caSink); readFile(actualPath, caSink);
}, },
[&](const FileIngestionMethod & m2) { [&](const FileIngestionMethod & m2) {

View file

@ -23,7 +23,7 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
std::string ContentAddressMethod::renderPrefix() const { std::string ContentAddressMethod::renderPrefix() const {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHashMethod) -> std::string { return "text:"; }, [](TextIngestionMethod) -> std::string { return "text:"; },
[](FileIngestionMethod m2) { [](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */ /* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2); return makeFileIngestionPrefix(m2);
@ -37,7 +37,7 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
if (splitPrefix(m, "r:")) if (splitPrefix(m, "r:"))
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
else if (splitPrefix(m, "text:")) else if (splitPrefix(m, "text:"))
method = TextHashMethod {}; method = TextIngestionMethod {};
return method; return method;
} }
@ -59,7 +59,7 @@ std::string ContentAddress::render() const
std::string ContentAddressMethod::render(HashType ht) const std::string ContentAddressMethod::render(HashType ht) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHashMethod & th) { [&](const TextIngestionMethod & th) {
return std::string{"text:"} + printHashType(ht); return std::string{"text:"} + printHashType(ht);
}, },
[&](const FileIngestionMethod & fim) { [&](const FileIngestionMethod & fim) {
@ -96,7 +96,7 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
// No parsing of the ingestion method, "text" only support flat. // No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
return { return {
TextHashMethod {}, TextIngestionMethod {},
std::move(hashType), std::move(hashType),
}; };
} else if (prefix == "fixed") { } else if (prefix == "fixed") {
@ -120,7 +120,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) {
auto hashType = hashType_; // work around clang bug auto hashType = hashType_; // work around clang bug
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod &) { [&](TextIngestionMethod &) {
return ContentAddress(TextHash { return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, hashType) .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
}); });
@ -158,7 +158,7 @@ ContentAddressWithReferences ContentAddressWithReferences::fromParts(
ContentAddressMethod method, Hash hash, StoreReferences refs) ContentAddressMethod method, Hash hash, StoreReferences refs)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod _) -> ContentAddressWithReferences { [&](TextIngestionMethod _) -> ContentAddressWithReferences {
if (refs.self) if (refs.self)
throw UsageError("Cannot have a self reference with text hashing scheme"); throw UsageError("Cannot have a self reference with text hashing scheme");
return TextInfo { return TextInfo {
@ -182,7 +182,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextInfo & th) -> ContentAddressMethod { [](const TextInfo & th) -> ContentAddressMethod {
return TextHashMethod {}; return TextIngestionMethod {};
}, },
[](const FixedOutputInfo & fsh) -> ContentAddressMethod { [](const FixedOutputInfo & fsh) -> ContentAddressMethod {
return fsh.hash.method; return fsh.hash.method;

View file

@ -22,7 +22,7 @@ namespace nix {
* Somewhat obscure, used by \ref Derivation derivations and * Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently. * `builtins.toFile` currently.
*/ */
struct TextHashMethod : std::monostate { }; struct TextIngestionMethod : std::monostate { };
/** /**
* An enumeration of the main ways we can serialize file system * An enumeration of the main ways we can serialize file system
@ -57,7 +57,7 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct ContentAddressMethod struct ContentAddressMethod
{ {
typedef std::variant< typedef std::variant<
TextHashMethod, TextIngestionMethod,
FileIngestionMethod FileIngestionMethod
> Raw; > Raw;

View file

@ -406,7 +406,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHashMethod &) { [&](const TextIngestionMethod &) {
if (hashType != htSHA256) if (hashType != htSHA256)
throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given", throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given",
printHashType(hashType)); printHashType(hashType));

View file

@ -216,6 +216,8 @@ static DerivationOutput parseDerivationOutput(const Store & store,
{ {
if (hashAlgo != "") { if (hashAlgo != "") {
ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
if (method == TextIngestionMethod {})
experimentalFeatureSettings.require(Xp::DynamicDerivations);
const auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hashS == "impure") { if (hashS == "impure") {
experimentalFeatureSettings.require(Xp::ImpureDerivations); experimentalFeatureSettings.require(Xp::ImpureDerivations);

View file

@ -339,12 +339,14 @@ struct Derivation : BasicDerivation
Store & store, Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
/* Check that the derivation is valid and does not present any /**
illegal states. * Check that the derivation is valid and does not present any
* illegal states.
This is mainly a matter of checking the outputs, where our C++ *
representation supports all sorts of combinations we do not yet * This is mainly a matter of checking the outputs, where our C++
allow. */ * representation supports all sorts of combinations we do not yet
* allow.
*/
void checkInvariants(Store & store, const StorePath & drvPath) const; void checkInvariants(Store & store, const StorePath & drvPath) const;
Derivation() = default; Derivation() = default;

View file

@ -629,7 +629,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
std::visit(overloaded { std::visit(overloaded {
[&](const TextHashMethod & thm) -> void { [&](const TextIngestionMethod & thm) -> void {
if (hashType != htSHA256) if (hashType != htSHA256)
throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given", throw UnimplementedError("Only SHA-256 is supported for adding text-hashed data, but '%1' was given",
printHashType(hashType)); printHashType(hashType));
@ -782,7 +782,7 @@ StorePath RemoteStore::addTextToStore(
RepairFlag repair) RepairFlag repair)
{ {
StringSource source(s); StringSource source(s);
return addCAToStore(source, name, TextHashMethod {}, htSHA256, references, repair)->path; return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
} }
void RemoteStore::registerDrvOutput(const Realisation & info) void RemoteStore::registerDrvOutput(const Realisation & info)

View file

@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
std::string_view description; std::string_view description;
}; };
constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
{ {
.tag = Xp::CaDerivations, .tag = Xp::CaDerivations,
.name = "ca-derivations", .name = "ca-derivations",
@ -199,6 +199,16 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
networking. networking.
)", )",
}, },
{
.tag = Xp::DynamicDerivations,
.name = "dynamic-derivations",
.description = R"(
Allow the use of a few things related to dynamic derivations:
- "text hashing" derivation outputs, so we can build .drv
files.
)",
},
}}; }};
static_assert( static_assert(

View file

@ -29,6 +29,7 @@ enum struct ExperimentalFeature
Cgroups, Cgroups,
DiscardReferences, DiscardReferences,
DaemonTrustOverride, DaemonTrustOverride,
DynamicDerivations,
}; };
/** /**

View file

@ -2,8 +2,10 @@
source common.sh source common.sh
# Globally enable the ca derivations experimental flag # Globally enable dynamic-derivations in addition to CA derivations
sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" enableFeatures "dynamic-derivations"
restartDaemon
# In the corresponding nix file, we have two derivations: the first, named root, # In the corresponding nix file, we have two derivations: the first, named root,
# is a normal recursive derivation, while the second, named dependent, has the # is a normal recursive derivation, while the second, named dependent, has the
@ -15,13 +17,13 @@ sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF
# - build the dependent derivation # - build the dependent derivation
# - check that the path of the output coincides with that of the original derivation # - check that the path of the output coincides with that of the original derivation
drv=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A root) drv=$(nix-instantiate ./text-hashed-output.nix -A root)
nix show-derivation "$drv" nix show-derivation "$drv"
drvDep=$(nix-instantiate --experimental-features ca-derivations ./text-hashed-output.nix -A dependent) drvDep=$(nix-instantiate ./text-hashed-output.nix -A dependent)
nix show-derivation "$drvDep" nix show-derivation "$drvDep"
out1=$(nix-build --experimental-features ca-derivations ./text-hashed-output.nix -A dependent --no-out-link) out1=$(nix-build ./text-hashed-output.nix -A dependent --no-out-link)
nix path-info $drv --derivation --json | jq nix path-info $drv --derivation --json | jq
nix path-info $out1 --derivation --json | jq nix path-info $out1 --derivation --json | jq