forked from lix-project/lix
Merge pull request #4628 from obsidiansystems/dynamic-drvs
Dynamic derivations RFC 92
This commit is contained in:
commit
e34493a70e
|
@ -2,8 +2,18 @@
|
|||
|
||||
For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
|
||||
|
||||
Derivations are serialised in the following format:
|
||||
Derivations are serialised in one of the following formats:
|
||||
|
||||
```
|
||||
- ```
|
||||
Derive(...)
|
||||
```
|
||||
|
||||
For all stable derivations.
|
||||
|
||||
- ```
|
||||
DrvWithVersion(<version-string>, ...)
|
||||
```
|
||||
|
||||
The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
|
||||
|
||||
- `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.
|
||||
|
|
|
@ -324,7 +324,7 @@ SV * derivationFromPath(char * drvPath)
|
|||
hv_stores(hash, "outputs", newRV((SV *) outputs));
|
||||
|
||||
AV * inputDrvs = newAV();
|
||||
for (auto & i : drv.inputDrvs)
|
||||
for (auto & i : drv.inputDrvs.map)
|
||||
av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
|
||||
hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
|
||||
|
||||
|
|
|
@ -314,7 +314,7 @@ connected:
|
|||
//
|
||||
// 2. Changing the `inputSrcs` set changes the associated
|
||||
// output ids, which break CA derivations
|
||||
if (!drv.inputDrvs.empty())
|
||||
if (!drv.inputDrvs.map.empty())
|
||||
drv.inputSrcs = store->parseStorePathSet(inputs);
|
||||
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
|
||||
auto & result = *optResult;
|
||||
|
|
|
@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const
|
|||
);
|
||||
}
|
||||
|
||||
SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
|
||||
{
|
||||
return SingleDerivedPath::Built {
|
||||
.drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
|
||||
.output = output.first,
|
||||
};
|
||||
}
|
||||
|
||||
SingleDerivedPath SingleBuiltPath::discardOutputPath() const
|
||||
{
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
|
||||
return p;
|
||||
},
|
||||
[](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
|
||||
return b.discardOutputPath();
|
||||
},
|
||||
}, raw()
|
||||
);
|
||||
}
|
||||
|
||||
nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
|
||||
{
|
||||
nlohmann::json res;
|
||||
|
|
|
@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt {
|
|||
ref<SingleBuiltPath> drvPath;
|
||||
std::pair<std::string, StorePath> output;
|
||||
|
||||
SingleDerivedPathBuilt discardOutputPath() const;
|
||||
|
||||
std::string to_string(const Store & store) const;
|
||||
static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
|
||||
nlohmann::json toJSON(const Store & store) const;
|
||||
|
@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
|
|||
|
||||
StorePath outPath() const;
|
||||
|
||||
SingleDerivedPath discardOutputPath() const;
|
||||
|
||||
static SingleBuiltPath parse(const Store & store, std::string_view);
|
||||
nlohmann::json toJSON(const Store & store) const;
|
||||
};
|
||||
|
|
|
@ -1252,15 +1252,13 @@ drvName, Bindings * attrs, Value & v)
|
|||
state.store->computeFSClosure(d.drvPath, refs);
|
||||
for (auto & j : refs) {
|
||||
drv.inputSrcs.insert(j);
|
||||
if (j.isDerivation())
|
||||
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
|
||||
if (j.isDerivation()) {
|
||||
drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames();
|
||||
}
|
||||
}
|
||||
},
|
||||
[&](const NixStringContextElem::Built & b) {
|
||||
if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
|
||||
drv.inputDrvs[p->path].insert(b.output);
|
||||
else
|
||||
throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
|
||||
drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
drv.inputSrcs.insert(o.path);
|
||||
|
|
|
@ -350,24 +350,36 @@ void DerivationGoal::gaveUpOnSubstitution()
|
|||
|
||||
/* The inputs must be built before we can build this goal. */
|
||||
inputDrvOutputs.clear();
|
||||
if (useDerivation)
|
||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
|
||||
if (useDerivation) {
|
||||
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
|
||||
|
||||
addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
if (!inputNode.value.empty())
|
||||
addWaitee(worker.makeGoal(
|
||||
DerivedPath::Built {
|
||||
.drvPath = inputDrv,
|
||||
.outputs = inputNode.value,
|
||||
},
|
||||
buildMode == bmRepair ? bmRepair : bmNormal));
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap)
|
||||
addWaiteeDerivedPath(
|
||||
make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
|
||||
childNode);
|
||||
};
|
||||
|
||||
for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
|
||||
/* Ensure that pure, non-fixed-output derivations don't
|
||||
depend on impure derivations. */
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
|
||||
auto inputDrv = worker.evalStore.readDerivation(i.first);
|
||||
auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
|
||||
if (!inputDrv.type().isPure())
|
||||
throw Error("pure derivation '%s' depends on impure derivation '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
worker.store.printStorePath(i.first));
|
||||
worker.store.printStorePath(inputDrvPath));
|
||||
}
|
||||
|
||||
addWaitee(worker.makeGoal(
|
||||
DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(i.first),
|
||||
.outputs = i.second,
|
||||
},
|
||||
buildMode == bmRepair ? bmRepair : bmNormal));
|
||||
addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
|
||||
}
|
||||
}
|
||||
|
||||
/* Copy the input sources from the eval store to the build
|
||||
|
@ -501,7 +513,7 @@ void DerivationGoal::inputsRealised()
|
|||
return ia.deferred;
|
||||
},
|
||||
[&](const DerivationType::ContentAddressed & ca) {
|
||||
return !fullDrv.inputDrvs.empty() && (
|
||||
return !fullDrv.inputDrvs.map.empty() && (
|
||||
ca.fixed
|
||||
/* Can optionally resolve if fixed, which is good
|
||||
for avoiding unnecessary rebuilds. */
|
||||
|
@ -515,7 +527,7 @@ void DerivationGoal::inputsRealised()
|
|||
}
|
||||
}, drvType.raw);
|
||||
|
||||
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
|
||||
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
|
||||
/* We are be able to resolve this derivation based on the
|
||||
|
@ -552,11 +564,13 @@ void DerivationGoal::inputsRealised()
|
|||
return;
|
||||
}
|
||||
|
||||
for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
|
||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
|
||||
|
||||
accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
/* Add the relevant output closures of the input derivation
|
||||
`i' as input paths. Only add the closures of output paths
|
||||
that are specified as inputs. */
|
||||
for (auto & j : wantedDepOutputs) {
|
||||
auto getOutput = [&](const std::string & outputName) {
|
||||
/* TODO (impure derivations-induced tech debt):
|
||||
Tracking input derivation outputs statefully through the
|
||||
goals is error prone and has led to bugs.
|
||||
|
@ -568,21 +582,30 @@ void DerivationGoal::inputsRealised()
|
|||
a representation in the store, which is a usability problem
|
||||
in itself. When implementing this logic entirely with lookups
|
||||
make sure that they're cached. */
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
|
||||
worker.store.computeFSClosure(*outPath, inputPaths);
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
|
||||
return *outPath;
|
||||
}
|
||||
else {
|
||||
auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
|
||||
auto outMapPath = outMap.find(j);
|
||||
auto outMapPath = outMap.find(outputName);
|
||||
if (outMapPath == outMap.end()) {
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
worker.store.computeFSClosure(outMapPath->second, inputPaths);
|
||||
}
|
||||
worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
return outMapPath->second;
|
||||
}
|
||||
};
|
||||
|
||||
for (auto & outputName : inputNode.value)
|
||||
worker.store.computeFSClosure(getOutput(outputName), inputPaths);
|
||||
|
||||
for (auto & [outputName, childNode] : inputNode.childMap)
|
||||
accumInputPaths(getOutput(outputName), childNode);
|
||||
};
|
||||
|
||||
for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
|
||||
accumInputPaths(depDrvPath, depNode);
|
||||
}
|
||||
|
||||
/* Second, the input sources. */
|
||||
|
@ -1475,22 +1498,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
|
|||
if (!useDerivation) return;
|
||||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
||||
|
||||
auto * dg = tryGetConcreteDrvGoal(waitee);
|
||||
if (!dg) return;
|
||||
std::optional info = tryGetConcreteDrvGoal(waitee);
|
||||
if (!info) return;
|
||||
const auto & [dg, drvReq] = *info;
|
||||
|
||||
auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
|
||||
if (outputs == fullDrv.inputDrvs.end()) return;
|
||||
auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
|
||||
if (!nodeP) return;
|
||||
auto & outputs = nodeP->value;
|
||||
|
||||
for (auto & outputName : outputs->second) {
|
||||
auto buildResult = dg->getBuildResult(DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(dg->drvPath),
|
||||
for (auto & outputName : outputs) {
|
||||
auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(dg.get().drvPath),
|
||||
.outputs = OutputsSpec::Names { outputName },
|
||||
});
|
||||
if (buildResult.success()) {
|
||||
auto i = buildResult.builtOutputs.find(outputName);
|
||||
if (i != buildResult.builtOutputs.end())
|
||||
inputDrvOutputs.insert_or_assign(
|
||||
{ dg->drvPath, outputName },
|
||||
{ dg.get().drvPath, outputName },
|
||||
i->second.outPath);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -594,11 +594,14 @@ GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
|
|||
return subGoal;
|
||||
}
|
||||
|
||||
const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
|
||||
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
|
||||
{
|
||||
auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
|
||||
if (!odg) return nullptr;
|
||||
return &*odg->concreteDrvGoal;
|
||||
if (!odg) return std::nullopt;
|
||||
return {{
|
||||
std::cref(*odg->concreteDrvGoal),
|
||||
std::cref(*odg->drvReq),
|
||||
}};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -49,7 +49,8 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
|||
* we have made the function, written in `worker.cc` where all the goal
|
||||
* types are visible, and use it instead.
|
||||
*/
|
||||
const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
|
||||
|
||||
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
|
||||
|
||||
/**
|
||||
* A mapping used to remember for each child process to what goal it
|
||||
|
|
|
@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store,
|
|||
const Derivation & drv, RepairFlag repair, bool readOnly)
|
||||
{
|
||||
auto references = drv.inputSrcs;
|
||||
for (auto & i : drv.inputDrvs)
|
||||
for (auto & i : drv.inputDrvs.map)
|
||||
references.insert(i.first);
|
||||
/* Note that the outputs of a derivation are *not* references
|
||||
(that can be missing (of course) and should not necessarily be
|
||||
|
@ -208,22 +208,25 @@ static bool endOfList(std::istream & str)
|
|||
static StringSet parseStrings(std::istream & str, bool arePaths)
|
||||
{
|
||||
StringSet res;
|
||||
expect(str, "[");
|
||||
while (!endOfList(str))
|
||||
res.insert(arePaths ? parsePath(str) : parseString(str));
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store,
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
|
||||
static DerivationOutput parseDerivationOutput(
|
||||
const Store & store,
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
|
||||
const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
if (hashAlgo != "") {
|
||||
ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
|
||||
if (method == TextIngestionMethod {})
|
||||
experimentalFeatureSettings.require(Xp::DynamicDerivations);
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
const auto hashType = parseHashType(hashAlgo);
|
||||
if (hashS == "impure") {
|
||||
experimentalFeatureSettings.require(Xp::ImpureDerivations);
|
||||
xpSettings.require(Xp::ImpureDerivations);
|
||||
if (pathS != "")
|
||||
throw FormatError("impure derivation output should not specify output path");
|
||||
return DerivationOutput::Impure {
|
||||
|
@ -240,7 +243,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
|||
},
|
||||
};
|
||||
} else {
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
xpSettings.require(Xp::CaDerivations);
|
||||
if (pathS != "")
|
||||
throw FormatError("content-addressed derivation output should not specify output path");
|
||||
return DerivationOutput::CAFloating {
|
||||
|
@ -259,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
|||
}
|
||||
}
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
|
||||
static DerivationOutput parseDerivationOutput(
|
||||
const Store & store, std::istringstream & str,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
|
||||
{
|
||||
expect(str, ","); const auto pathS = parseString(str);
|
||||
expect(str, ","); const auto hashAlgo = parseString(str);
|
||||
expect(str, ","); const auto hash = parseString(str);
|
||||
expect(str, ")");
|
||||
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
|
||||
}
|
||||
|
||||
/**
|
||||
* All ATerm Derivation format versions currently known.
|
||||
*
|
||||
* Unknown versions are rejected at the parsing stage.
|
||||
*/
|
||||
enum struct DerivationATermVersion {
|
||||
/**
|
||||
* Older unversioned form
|
||||
*/
|
||||
Traditional,
|
||||
|
||||
/**
|
||||
* Newer versioned form; only this version so far.
|
||||
*/
|
||||
DynamicDerivations,
|
||||
};
|
||||
|
||||
static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
|
||||
const Store & store,
|
||||
std::istringstream & str,
|
||||
DerivationATermVersion version)
|
||||
{
|
||||
DerivedPathMap<StringSet>::ChildNode node;
|
||||
|
||||
auto parseNonDynamic = [&]() {
|
||||
node.value = parseStrings(str, false);
|
||||
};
|
||||
|
||||
// Older derivation should never use new form, but newer
|
||||
// derivaiton can use old form.
|
||||
switch (version) {
|
||||
case DerivationATermVersion::Traditional:
|
||||
parseNonDynamic();
|
||||
break;
|
||||
case DerivationATermVersion::DynamicDerivations:
|
||||
switch (str.peek()) {
|
||||
case '[':
|
||||
parseNonDynamic();
|
||||
break;
|
||||
case '(':
|
||||
expect(str, "(");
|
||||
node.value = parseStrings(str, false);
|
||||
expect(str, ",[");
|
||||
while (!endOfList(str)) {
|
||||
expect(str, "(");
|
||||
auto outputName = parseString(str);
|
||||
expect(str, ",");
|
||||
node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
|
||||
expect(str, ")");
|
||||
}
|
||||
expect(str, ")");
|
||||
break;
|
||||
default:
|
||||
throw FormatError("invalid inputDrvs entry in derivation");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// invalid format, not a parse error but internal error
|
||||
assert(false);
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
|
||||
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
|
||||
Derivation parseDerivation(
|
||||
const Store & store, std::string && s, std::string_view name,
|
||||
const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
Derivation drv;
|
||||
drv.name = name;
|
||||
|
||||
std::istringstream str(std::move(s));
|
||||
expect(str, "Derive([");
|
||||
expect(str, "D");
|
||||
DerivationATermVersion version;
|
||||
switch (str.peek()) {
|
||||
case 'e':
|
||||
expect(str, "erive(");
|
||||
version = DerivationATermVersion::Traditional;
|
||||
break;
|
||||
case 'r':
|
||||
expect(str, "rvWithVersion(");
|
||||
auto versionS = parseString(str);
|
||||
if (versionS == "xp-dyn-drv") {
|
||||
// Only verison we have so far
|
||||
version = DerivationATermVersion::DynamicDerivations;
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
} else {
|
||||
throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
|
||||
}
|
||||
expect(str, ",");
|
||||
break;
|
||||
}
|
||||
|
||||
/* Parse the list of outputs. */
|
||||
expect(str, "[");
|
||||
while (!endOfList(str)) {
|
||||
expect(str, "("); std::string id = parseString(str);
|
||||
auto output = parseDerivationOutput(store, str);
|
||||
auto output = parseDerivationOutput(store, str, xpSettings);
|
||||
drv.outputs.emplace(std::move(id), std::move(output));
|
||||
}
|
||||
|
||||
|
@ -290,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
|
|||
while (!endOfList(str)) {
|
||||
expect(str, "(");
|
||||
Path drvPath = parsePath(str);
|
||||
expect(str, ",[");
|
||||
drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
|
||||
expect(str, ",");
|
||||
drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
|
||||
expect(str, ")");
|
||||
}
|
||||
|
||||
expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
|
||||
expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
|
||||
expect(str, ","); drv.platform = parseString(str);
|
||||
expect(str, ","); drv.builder = parseString(str);
|
||||
|
||||
|
@ -379,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
|
|||
}
|
||||
|
||||
|
||||
static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
|
||||
{
|
||||
s += ',';
|
||||
if (node.childMap.empty()) {
|
||||
printUnquotedStrings(s, node.value.begin(), node.value.end());
|
||||
} else {
|
||||
s += "(";
|
||||
printUnquotedStrings(s, node.value.begin(), node.value.end());
|
||||
s += ",[";
|
||||
bool first = true;
|
||||
for (auto & [outputName, childNode] : node.childMap) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, outputName);
|
||||
unparseDerivedPathMapNode(store, s, childNode);
|
||||
s += ')';
|
||||
}
|
||||
s += "])";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Does the derivation have a dependency on the output of a dynamic
|
||||
* derivation?
|
||||
*
|
||||
* In other words, does it on the output of derivation that is itself an
|
||||
* ouput of a derivation? This corresponds to a dependency that is an
|
||||
* inductive derived path with more than one layer of
|
||||
* `DerivedPath::Built`.
|
||||
*/
|
||||
static bool hasDynamicDrvDep(const Derivation & drv)
|
||||
{
|
||||
return
|
||||
std::find_if(
|
||||
drv.inputDrvs.map.begin(),
|
||||
drv.inputDrvs.map.end(),
|
||||
[](auto & kv) { return !kv.second.childMap.empty(); })
|
||||
!= drv.inputDrvs.map.end();
|
||||
}
|
||||
|
||||
|
||||
std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
||||
std::map<std::string, StringSet> * actualInputs) const
|
||||
DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
|
||||
{
|
||||
std::string s;
|
||||
s.reserve(65536);
|
||||
s += "Derive([";
|
||||
|
||||
/* Use older unversioned form if possible, for wider compat. Use
|
||||
newer form only if we need it, which we do for
|
||||
`Xp::DynamicDerivations`. */
|
||||
if (hasDynamicDrvDep(*this)) {
|
||||
s += "DrvWithVersion(";
|
||||
// Only version we have so far
|
||||
printUnquotedString(s, "xp-dyn-drv");
|
||||
s += ",";
|
||||
} else {
|
||||
s += "Derive(";
|
||||
}
|
||||
|
||||
bool first = true;
|
||||
s += "[";
|
||||
for (auto & i : outputs) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, i.first);
|
||||
|
@ -424,17 +567,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
s += "],[";
|
||||
first = true;
|
||||
if (actualInputs) {
|
||||
for (auto & i : *actualInputs) {
|
||||
for (auto & [drvHashModulo, childMap] : *actualInputs) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, i.first);
|
||||
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
|
||||
s += '('; printUnquotedString(s, drvHashModulo);
|
||||
unparseDerivedPathMapNode(store, s, childMap);
|
||||
s += ')';
|
||||
}
|
||||
} else {
|
||||
for (auto & i : inputDrvs) {
|
||||
for (auto & [drvPath, childMap] : inputDrvs.map) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, store.printStorePath(i.first));
|
||||
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
|
||||
s += '('; printUnquotedString(s, store.printStorePath(drvPath));
|
||||
unparseDerivedPathMapNode(store, s, childMap);
|
||||
s += ')';
|
||||
}
|
||||
}
|
||||
|
@ -668,18 +811,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
|
|||
}
|
||||
}, drv.type().raw);
|
||||
|
||||
std::map<std::string, StringSet> inputs2;
|
||||
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
|
||||
// Avoid lambda capture restriction with standard / Clang
|
||||
auto & inputOutputs = inputOutputs0;
|
||||
DerivedPathMap<StringSet>::ChildNode::Map inputs2;
|
||||
for (auto & [drvPath, node] : drv.inputDrvs.map) {
|
||||
const auto & res = pathDerivationModulo(store, drvPath);
|
||||
if (res.kind == DrvHash::Kind::Deferred)
|
||||
kind = DrvHash::Kind::Deferred;
|
||||
for (auto & outputName : inputOutputs) {
|
||||
for (auto & outputName : node.value) {
|
||||
const auto h = get(res.hashes, outputName);
|
||||
if (!h)
|
||||
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
||||
inputs2[h->to_string(Base16, false)].insert(outputName);
|
||||
inputs2[h->to_string(Base16, false)].value.insert(outputName);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -709,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
|||
const auto hashAlgo = readString(in);
|
||||
const auto hash = readString(in);
|
||||
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings);
|
||||
}
|
||||
|
||||
StringSet BasicDerivation::outputNames() const
|
||||
|
@ -824,6 +965,8 @@ std::string hashPlaceholder(const OutputNameView outputName)
|
|||
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||
{
|
||||
debug("Rewriting the derivation");
|
||||
|
||||
for (auto & rewrite : rewrites) {
|
||||
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
||||
}
|
||||
|
@ -862,14 +1005,70 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
|
|||
{
|
||||
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||
|
||||
for (auto & input : inputDrvs)
|
||||
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
|
||||
if (outputPath)
|
||||
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
|
||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
|
||||
accum = [&](auto & inputDrv, auto & node) {
|
||||
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
|
||||
if (outputPath) {
|
||||
inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
|
||||
if (auto p = get(node.childMap, outputName))
|
||||
accum(*outputPath, *p);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (auto & [inputDrv, node] : inputDrvs.map)
|
||||
accum(inputDrv, node);
|
||||
|
||||
return tryResolve(store, inputDrvOutputs);
|
||||
}
|
||||
|
||||
static bool tryResolveInput(
|
||||
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
|
||||
const DownstreamPlaceholder * placeholderOpt,
|
||||
const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode,
|
||||
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs)
|
||||
{
|
||||
auto getOutput = [&](const std::string & outputName) {
|
||||
auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName });
|
||||
if (!actualPathOpt)
|
||||
warn("output %s of input %s missing, aborting the resolving",
|
||||
outputName,
|
||||
store.printStorePath(inputDrv)
|
||||
);
|
||||
return actualPathOpt;
|
||||
};
|
||||
|
||||
auto getPlaceholder = [&](const std::string & outputName) {
|
||||
return placeholderOpt
|
||||
? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName)
|
||||
: DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName);
|
||||
};
|
||||
|
||||
for (auto & outputName : inputNode.value) {
|
||||
auto actualPathOpt = getOutput(outputName);
|
||||
if (!actualPathOpt) return false;
|
||||
auto actualPath = *actualPathOpt;
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
inputRewrites.emplace(
|
||||
getPlaceholder(outputName).render(),
|
||||
store.printStorePath(actualPath));
|
||||
}
|
||||
inputSrcs.insert(std::move(actualPath));
|
||||
}
|
||||
|
||||
for (auto & [outputName, childNode] : inputNode.childMap) {
|
||||
auto actualPathOpt = getOutput(outputName);
|
||||
if (!actualPathOpt) return false;
|
||||
auto actualPath = *actualPathOpt;
|
||||
auto nextPlaceholder = getPlaceholder(outputName);
|
||||
if (!tryResolveInput(store, inputSrcs, inputRewrites,
|
||||
&nextPlaceholder, actualPath, childNode,
|
||||
inputDrvOutputs))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
Store & store,
|
||||
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
|
||||
|
@ -879,23 +1078,10 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
|||
// Input paths that we'll want to rewrite in the derivation
|
||||
StringMap inputRewrites;
|
||||
|
||||
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
|
||||
for (auto & outputName : inputOutputs) {
|
||||
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
inputRewrites.emplace(
|
||||
DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
|
||||
store.printStorePath(*actualPath));
|
||||
}
|
||||
resolved.inputSrcs.insert(*actualPath);
|
||||
} else {
|
||||
warn("output '%s' of input '%s' missing, aborting the resolving",
|
||||
outputName,
|
||||
store.printStorePath(inputDrv));
|
||||
return {};
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto & [inputDrv, inputNode] : inputDrvs.map)
|
||||
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites,
|
||||
nullptr, inputDrv, inputNode, inputDrvOutputs))
|
||||
return std::nullopt;
|
||||
|
||||
rewriteDerivation(store, resolved, inputRewrites);
|
||||
|
||||
|
@ -1083,11 +1269,26 @@ nlohmann::json Derivation::toJSON(const Store & store) const
|
|||
inputsList.emplace_back(store.printStorePath(input));
|
||||
}
|
||||
|
||||
{
|
||||
std::function<nlohmann::json(const DerivedPathMap<StringSet>::ChildNode &)> doInput;
|
||||
doInput = [&](const auto & inputNode) {
|
||||
auto value = nlohmann::json::object();
|
||||
value["outputs"] = inputNode.value;
|
||||
{
|
||||
auto next = nlohmann::json::object();
|
||||
for (auto & [outputId, childNode] : inputNode.childMap)
|
||||
next[outputId] = doInput(childNode);
|
||||
value["dynamicOutputs"] = std::move(next);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
{
|
||||
auto& inputDrvsObj = res["inputDrvs"];
|
||||
inputDrvsObj = nlohmann::json::object();
|
||||
for (auto & input : inputDrvs)
|
||||
inputDrvsObj[store.printStorePath(input.first)] = input.second;
|
||||
for (auto & [inputDrv, inputNode] : inputDrvs.map) {
|
||||
inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res["system"] = platform;
|
||||
|
@ -1101,7 +1302,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const
|
|||
|
||||
Derivation Derivation::fromJSON(
|
||||
const Store & store,
|
||||
const nlohmann::json & json)
|
||||
const nlohmann::json & json,
|
||||
const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
using nlohmann::detail::value_t;
|
||||
|
||||
|
@ -1133,12 +1335,21 @@ Derivation Derivation::fromJSON(
|
|||
}
|
||||
|
||||
try {
|
||||
auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
|
||||
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) {
|
||||
ensureType(inputOutputs, value_t::array);
|
||||
res.inputDrvs[store.parseStorePath(inputDrvPath)] =
|
||||
static_cast<const StringSet &>(inputOutputs);
|
||||
std::function<DerivedPathMap<StringSet>::ChildNode(const nlohmann::json &)> doInput;
|
||||
doInput = [&](const auto & json) {
|
||||
DerivedPathMap<StringSet>::ChildNode node;
|
||||
node.value = static_cast<const StringSet &>(
|
||||
ensureType(valueAt(json, "outputs"), value_t::array));
|
||||
for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) {
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
node.childMap[outputId] = doInput(childNode);
|
||||
}
|
||||
return node;
|
||||
};
|
||||
auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
|
||||
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
|
||||
res.inputDrvs.map[store.parseStorePath(inputDrvPath)] =
|
||||
doInput(inputOutputs);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading key 'inputDrvs'");
|
||||
throw;
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
#include "hash.hh"
|
||||
#include "content-address.hh"
|
||||
#include "repair-flag.hh"
|
||||
#include "derived-path.hh"
|
||||
#include "derived-path-map.hh"
|
||||
#include "sync.hh"
|
||||
#include "comparator.hh"
|
||||
#include "variant-wrapper.hh"
|
||||
|
@ -323,13 +323,13 @@ struct Derivation : BasicDerivation
|
|||
/**
|
||||
* inputs that are sub-derivations
|
||||
*/
|
||||
DerivationInputs inputDrvs;
|
||||
DerivedPathMap<std::set<OutputName>> inputDrvs;
|
||||
|
||||
/**
|
||||
* Print a derivation.
|
||||
*/
|
||||
std::string unparse(const Store & store, bool maskOutputs,
|
||||
std::map<std::string, StringSet> * actualInputs = nullptr) const;
|
||||
DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
|
||||
|
||||
/**
|
||||
* Return the underlying basic derivation but with these changes:
|
||||
|
@ -368,7 +368,8 @@ struct Derivation : BasicDerivation
|
|||
nlohmann::json toJSON(const Store & store) const;
|
||||
static Derivation fromJSON(
|
||||
const Store & store,
|
||||
const nlohmann::json & json);
|
||||
const nlohmann::json & json,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
GENERATE_CMP(Derivation,
|
||||
static_cast<const BasicDerivation &>(*me),
|
||||
|
@ -389,7 +390,11 @@ StorePath writeDerivation(Store & store,
|
|||
/**
|
||||
* Read a derivation from a file.
|
||||
*/
|
||||
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
|
||||
Derivation parseDerivation(
|
||||
const Store & store,
|
||||
std::string && s,
|
||||
std::string_view name,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
/**
|
||||
* \todo Remove.
|
||||
|
|
|
@ -21,6 +21,32 @@ typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const Sing
|
|||
return initIter(k);
|
||||
}
|
||||
|
||||
template<typename V>
|
||||
typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const SingleDerivedPath & k)
|
||||
{
|
||||
std::function<ChildNode *(const SingleDerivedPath & )> initIter;
|
||||
initIter = [&](const auto & k) {
|
||||
return std::visit(overloaded {
|
||||
[&](const SingleDerivedPath::Opaque & bo) {
|
||||
auto it = map.find(bo.path);
|
||||
return it != map.end()
|
||||
? &it->second
|
||||
: nullptr;
|
||||
},
|
||||
[&](const SingleDerivedPath::Built & bfd) {
|
||||
auto * n = initIter(*bfd.drvPath);
|
||||
if (!n) return (ChildNode *)nullptr;
|
||||
|
||||
auto it = n->childMap.find(bfd.output);
|
||||
return it != n->childMap.end()
|
||||
? &it->second
|
||||
: nullptr;
|
||||
},
|
||||
}, k.raw());
|
||||
};
|
||||
return initIter(k);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// instantiations
|
||||
|
@ -30,4 +56,17 @@ namespace nix {
|
|||
|
||||
template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
|
||||
|
||||
}
|
||||
GENERATE_CMP_EXT(
|
||||
template<>,
|
||||
DerivedPathMap<std::set<std::string>>::ChildNode,
|
||||
me->value,
|
||||
me->childMap);
|
||||
|
||||
GENERATE_CMP_EXT(
|
||||
template<>,
|
||||
DerivedPathMap<std::set<std::string>>,
|
||||
me->map);
|
||||
|
||||
template struct DerivedPathMap<std::set<std::string>>;
|
||||
|
||||
};
|
||||
|
|
|
@ -48,6 +48,8 @@ struct DerivedPathMap {
|
|||
* The map of the root node.
|
||||
*/
|
||||
Map childMap;
|
||||
|
||||
DECLARE_CMP(ChildNode);
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -60,6 +62,8 @@ struct DerivedPathMap {
|
|||
*/
|
||||
Map map;
|
||||
|
||||
DECLARE_CMP(DerivedPathMap);
|
||||
|
||||
/**
|
||||
* Find the node for `k`, creating it if needed.
|
||||
*
|
||||
|
@ -68,6 +72,27 @@ struct DerivedPathMap {
|
|||
* by changing this node.
|
||||
*/
|
||||
ChildNode & ensureSlot(const SingleDerivedPath & k);
|
||||
|
||||
/**
|
||||
* Like `ensureSlot` but does not create the slot if it doesn't exist.
|
||||
*
|
||||
* Read the entire description of `ensureSlot` to understand an
|
||||
* important caveat here that "have slot" does *not* imply "key is
|
||||
* set in map". To ensure a key is set one would need to get the
|
||||
* child node (with `findSlot` or `ensureSlot`) *and* check the
|
||||
* `ChildNode::value`.
|
||||
*/
|
||||
ChildNode * findSlot(const SingleDerivedPath & k);
|
||||
};
|
||||
|
||||
|
||||
DECLARE_CMP_EXT(
|
||||
template<>,
|
||||
DerivedPathMap<std::set<std::string>>::,
|
||||
DerivedPathMap<std::set<std::string>>);
|
||||
DECLARE_CMP_EXT(
|
||||
template<>,
|
||||
DerivedPathMap<std::set<std::string>>::ChildNode::,
|
||||
DerivedPathMap<std::set<std::string>>::ChildNode);
|
||||
|
||||
}
|
||||
|
|
|
@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
|
||||
std::function<void(DerivedPath)> doPath;
|
||||
|
||||
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> enqueueDerivedPaths;
|
||||
|
||||
enqueueDerivedPaths = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
if (!inputNode.value.empty())
|
||||
pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value }));
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap)
|
||||
enqueueDerivedPaths(
|
||||
make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
|
||||
childNode);
|
||||
};
|
||||
|
||||
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->willBuild.insert(drvPath);
|
||||
}
|
||||
|
||||
for (auto & i : drv.inputDrvs)
|
||||
pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second }));
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) {
|
||||
enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode);
|
||||
}
|
||||
};
|
||||
|
||||
auto checkOutput = [&](
|
||||
|
@ -322,10 +334,13 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
|||
{
|
||||
std::set<Realisation> inputRealisations;
|
||||
|
||||
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
|
||||
const auto outputHashes =
|
||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
|
||||
|
||||
accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
if (!inputNode.value.empty()) {
|
||||
auto outputHashes =
|
||||
staticOutputHashes(store, store.readDerivation(inputDrv));
|
||||
for (const auto & outputName : outputNames) {
|
||||
for (const auto & outputName : inputNode.value) {
|
||||
auto outputHash = get(outputHashes, outputName);
|
||||
if (!outputHash)
|
||||
throw Error(
|
||||
|
@ -335,11 +350,25 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
|||
DrvOutput{*outputHash, outputName});
|
||||
if (!thisRealisation)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn't built", outputName,
|
||||
"output '%s' of derivation '%s' isn’t built", outputName,
|
||||
store.printStorePath(inputDrv));
|
||||
inputRealisations.insert(*thisRealisation);
|
||||
}
|
||||
}
|
||||
if (!inputNode.value.empty()) {
|
||||
auto d = makeConstantStorePathRef(inputDrv);
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap) {
|
||||
SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
|
||||
accumRealisations(
|
||||
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
||||
resolveDerivedPath(store, next),
|
||||
childNode);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
||||
accumRealisations(inputDrv, inputNode);
|
||||
|
||||
auto info = store.queryPathInfo(outputPath);
|
||||
|
||||
|
|
|
@ -172,7 +172,24 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source,
|
|||
auto ex = handle->processStderr(sink, source, flush);
|
||||
if (ex) {
|
||||
daemonException = true;
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (const Error & e) {
|
||||
// Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded.
|
||||
// To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the
|
||||
// old incomprehensible error here, so that we can explain to users what's going on when their daemon is
|
||||
// older than #4628 (2023).
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) &&
|
||||
GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35)
|
||||
{
|
||||
auto m = e.msg();
|
||||
if (m.find("parsing derivation") != std::string::npos &&
|
||||
m.find("expected string") != std::string::npos &&
|
||||
m.find("Derive([") != std::string::npos)
|
||||
throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -42,6 +42,26 @@ class ImpureDerivationTest : public DerivationTest
|
|||
}
|
||||
};
|
||||
|
||||
TEST_F(DerivationTest, BadATerm_version) {
|
||||
ASSERT_THROW(
|
||||
parseDerivation(
|
||||
*store,
|
||||
R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
|
||||
"whatever",
|
||||
mockXpSettings),
|
||||
FormatError);
|
||||
}
|
||||
|
||||
TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
|
||||
ASSERT_THROW(
|
||||
parseDerivation(
|
||||
*store,
|
||||
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
|
||||
"dyn-dep-derivation",
|
||||
mockXpSettings),
|
||||
FormatError);
|
||||
}
|
||||
|
||||
#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
|
||||
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \
|
||||
using nlohmann::literals::operator "" _json; \
|
||||
|
@ -143,35 +163,37 @@ TEST_JSON(ImpureDerivationTest, impure,
|
|||
|
||||
#undef TEST_JSON
|
||||
|
||||
#define TEST_JSON(NAME, STR, VAL) \
|
||||
TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \
|
||||
#define TEST_JSON(FIXTURE, NAME, STR, VAL) \
|
||||
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \
|
||||
using nlohmann::literals::operator "" _json; \
|
||||
ASSERT_EQ( \
|
||||
STR ## _json, \
|
||||
(VAL).toJSON(*store)); \
|
||||
} \
|
||||
\
|
||||
TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \
|
||||
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \
|
||||
using nlohmann::literals::operator "" _json; \
|
||||
ASSERT_EQ( \
|
||||
(VAL), \
|
||||
Derivation::fromJSON( \
|
||||
*store, \
|
||||
STR ## _json)); \
|
||||
STR ## _json, \
|
||||
mockXpSettings)); \
|
||||
}
|
||||
|
||||
#define TEST_ATERM(NAME, STR, VAL, DRV_NAME) \
|
||||
TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) { \
|
||||
#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \
|
||||
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \
|
||||
ASSERT_EQ( \
|
||||
STR, \
|
||||
(VAL).unparse(*store, false)); \
|
||||
} \
|
||||
\
|
||||
TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \
|
||||
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \
|
||||
auto parsed = parseDerivation( \
|
||||
*store, \
|
||||
STR, \
|
||||
DRV_NAME); \
|
||||
DRV_NAME, \
|
||||
mockXpSettings); \
|
||||
ASSERT_EQ( \
|
||||
(VAL).toJSON(*store), \
|
||||
parsed.toJSON(*store)); \
|
||||
|
@ -187,13 +209,17 @@ Derivation makeSimpleDrv(const Store & store) {
|
|||
store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
|
||||
};
|
||||
drv.inputDrvs = {
|
||||
.map = {
|
||||
{
|
||||
store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
|
||||
{
|
||||
.value = {
|
||||
"cat",
|
||||
"dog",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
drv.platform = "wasm-sel4";
|
||||
drv.builder = "foo";
|
||||
|
@ -210,17 +236,20 @@ Derivation makeSimpleDrv(const Store & store) {
|
|||
return drv;
|
||||
}
|
||||
|
||||
TEST_JSON(simple,
|
||||
TEST_JSON(DerivationTest, simple,
|
||||
R"({
|
||||
"name": "simple-derivation",
|
||||
"inputSrcs": [
|
||||
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
|
||||
],
|
||||
"inputDrvs": {
|
||||
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [
|
||||
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": [
|
||||
"cat",
|
||||
"dog"
|
||||
]
|
||||
}
|
||||
},
|
||||
"system": "wasm-sel4",
|
||||
"builder": "foo",
|
||||
|
@ -235,11 +264,105 @@ TEST_JSON(simple,
|
|||
})",
|
||||
makeSimpleDrv(*store))
|
||||
|
||||
TEST_ATERM(simple,
|
||||
TEST_ATERM(DerivationTest, simple,
|
||||
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
|
||||
makeSimpleDrv(*store),
|
||||
"simple-derivation")
|
||||
|
||||
Derivation makeDynDepDerivation(const Store & store) {
|
||||
Derivation drv;
|
||||
drv.name = "dyn-dep-derivation";
|
||||
drv.inputSrcs = {
|
||||
store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
|
||||
};
|
||||
drv.inputDrvs = {
|
||||
.map = {
|
||||
{
|
||||
store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
|
||||
DerivedPathMap<StringSet>::ChildNode {
|
||||
.value = {
|
||||
"cat",
|
||||
"dog",
|
||||
},
|
||||
.childMap = {
|
||||
{
|
||||
"cat",
|
||||
DerivedPathMap<StringSet>::ChildNode {
|
||||
.value = {
|
||||
"kitten",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"goose",
|
||||
DerivedPathMap<StringSet>::ChildNode {
|
||||
.value = {
|
||||
"gosling",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
drv.platform = "wasm-sel4";
|
||||
drv.builder = "foo";
|
||||
drv.args = {
|
||||
"bar",
|
||||
"baz",
|
||||
};
|
||||
drv.env = {
|
||||
{
|
||||
"BIG_BAD",
|
||||
"WOLF",
|
||||
},
|
||||
};
|
||||
return drv;
|
||||
}
|
||||
|
||||
TEST_JSON(DynDerivationTest, dynDerivationDeps,
|
||||
R"({
|
||||
"name": "dyn-dep-derivation",
|
||||
"inputSrcs": [
|
||||
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
|
||||
],
|
||||
"inputDrvs": {
|
||||
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
|
||||
"dynamicOutputs": {
|
||||
"cat": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": ["kitten"]
|
||||
},
|
||||
"goose": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": ["gosling"]
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
"cat",
|
||||
"dog"
|
||||
]
|
||||
}
|
||||
},
|
||||
"system": "wasm-sel4",
|
||||
"builder": "foo",
|
||||
"args": [
|
||||
"bar",
|
||||
"baz"
|
||||
],
|
||||
"env": {
|
||||
"BIG_BAD": "WOLF"
|
||||
},
|
||||
"outputs": {}
|
||||
})",
|
||||
makeDynDepDerivation(*store))
|
||||
|
||||
TEST_ATERM(DynDerivationTest, dynDerivationDeps,
|
||||
R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
|
||||
makeDynDepDerivation(*store),
|
||||
"dyn-dep-derivation")
|
||||
|
||||
#undef TEST_JSON
|
||||
#undef TEST_ATERM
|
||||
|
||||
|
|
|
@ -1,21 +1,48 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \
|
||||
PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const;
|
||||
#define DECLARE_EQUAL(prefix, qualification, my_type) \
|
||||
DECLARE_ONE_CMP(prefix, qualification, ==, my_type)
|
||||
#define DECLARE_LEQ(prefix, qualification, my_type) \
|
||||
DECLARE_ONE_CMP(prefix, qualification, <, my_type)
|
||||
#define DECLARE_NEQ(prefix, qualification, my_type) \
|
||||
DECLARE_ONE_CMP(prefix, qualification, !=, my_type)
|
||||
|
||||
#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
|
||||
PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
|
||||
__VA_OPT__(const MY_TYPE * me = this;) \
|
||||
auto fields1 = std::make_tuple( __VA_ARGS__ ); \
|
||||
__VA_OPT__(me = &other;) \
|
||||
auto fields2 = std::make_tuple( __VA_ARGS__ ); \
|
||||
return fields1 COMPARATOR fields2; \
|
||||
}
|
||||
#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \
|
||||
GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args)
|
||||
#define GENERATE_LEQ(prefix, qualification, my_type, args...) \
|
||||
GENERATE_ONE_CMP(prefix, qualification, <, my_type, args)
|
||||
#define GENERATE_NEQ(prefix, qualification, my_type, args...) \
|
||||
GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args)
|
||||
|
||||
/**
|
||||
* Declare comparison methods without defining them.
|
||||
*/
|
||||
#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \
|
||||
bool operator COMPARATOR(const MY_TYPE & other) const;
|
||||
#define DECLARE_EQUAL(my_type) \
|
||||
DECLARE_ONE_CMP(==, my_type)
|
||||
#define DECLARE_LEQ(my_type) \
|
||||
DECLARE_ONE_CMP(<, my_type)
|
||||
#define DECLARE_NEQ(my_type) \
|
||||
DECLARE_ONE_CMP(!=, my_type)
|
||||
#define DECLARE_CMP(my_type) \
|
||||
DECLARE_EQUAL(my_type) \
|
||||
DECLARE_LEQ(my_type) \
|
||||
DECLARE_NEQ(my_type)
|
||||
DECLARE_EQUAL(,,my_type) \
|
||||
DECLARE_LEQ(,,my_type) \
|
||||
DECLARE_NEQ(,,my_type)
|
||||
|
||||
/**
|
||||
* @param prefix This is for something before each declaration like
|
||||
* `template<classname Foo>`.
|
||||
*
|
||||
* @param my_type the type are defining operators for.
|
||||
*/
|
||||
#define DECLARE_CMP_EXT(prefix, qualification, my_type) \
|
||||
DECLARE_EQUAL(prefix, qualification, my_type) \
|
||||
DECLARE_LEQ(prefix, qualification, my_type) \
|
||||
DECLARE_NEQ(prefix, qualification, my_type)
|
||||
|
||||
/**
|
||||
* Awful hacky generation of the comparison operators by doing a lexicographic
|
||||
|
@ -33,18 +60,18 @@
|
|||
* }
|
||||
* ```
|
||||
*/
|
||||
#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \
|
||||
bool operator COMPARATOR(const MY_TYPE& other) const { \
|
||||
__VA_OPT__(const MY_TYPE* me = this;) \
|
||||
auto fields1 = std::make_tuple( __VA_ARGS__ ); \
|
||||
__VA_OPT__(me = &other;) \
|
||||
auto fields2 = std::make_tuple( __VA_ARGS__ ); \
|
||||
return fields1 COMPARATOR fields2; \
|
||||
}
|
||||
#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
|
||||
#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
|
||||
#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
|
||||
#define GENERATE_CMP(args...) \
|
||||
GENERATE_EQUAL(args) \
|
||||
GENERATE_LEQ(args) \
|
||||
GENERATE_NEQ(args)
|
||||
GENERATE_EQUAL(,,args) \
|
||||
GENERATE_LEQ(,,args) \
|
||||
GENERATE_NEQ(,,args)
|
||||
|
||||
/**
|
||||
* @param prefix This is for something before each declaration like
|
||||
* `template<classname Foo>`.
|
||||
*
|
||||
* @param my_type the type are defining operators for.
|
||||
*/
|
||||
#define GENERATE_CMP_EXT(prefix, my_type, args...) \
|
||||
GENERATE_EQUAL(prefix, my_type ::, my_type, args) \
|
||||
GENERATE_LEQ(prefix, my_type ::, my_type, args) \
|
||||
GENERATE_NEQ(prefix, my_type ::, my_type, args)
|
||||
|
|
|
@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv)
|
|||
}
|
||||
}
|
||||
|
||||
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
|
||||
|
||||
accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
if (!inputNode.value.empty())
|
||||
pathsToBuild.push_back(DerivedPath::Built {
|
||||
.drvPath = inputDrv,
|
||||
.outputs = OutputsSpec::Names { inputNode.value },
|
||||
});
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap)
|
||||
accumDerivedPath(
|
||||
make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
|
||||
childNode);
|
||||
};
|
||||
|
||||
// Build or fetch all dependencies of the derivation.
|
||||
for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) {
|
||||
for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
|
||||
// To get around lambda capturing restrictions in the
|
||||
// standard.
|
||||
const auto & inputDrv = inputDrv0;
|
||||
|
@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv)
|
|||
return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude));
|
||||
}))
|
||||
{
|
||||
pathsToBuild.push_back(DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(inputDrv),
|
||||
.outputs = OutputsSpec::Names { inputOutputs },
|
||||
});
|
||||
accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
|
||||
pathsToCopy.insert(inputDrv);
|
||||
}
|
||||
}
|
||||
|
@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv)
|
|||
|
||||
if (env.count("__json")) {
|
||||
StorePathSet inputs;
|
||||
for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) {
|
||||
auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath);
|
||||
for (auto & i : wantedDepOutputs) {
|
||||
|
||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
|
||||
|
||||
accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||
auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv);
|
||||
for (auto & i : inputNode.value) {
|
||||
auto o = outputs.at(i);
|
||||
store->computeFSClosure(*o, inputs);
|
||||
}
|
||||
}
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap)
|
||||
accumInputClosure(*outputs.at(outputName), childNode);
|
||||
};
|
||||
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
||||
accumInputClosure(inputDrv, inputNode);
|
||||
|
||||
ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv);
|
||||
|
||||
|
|
|
@ -20,15 +20,22 @@ StringPairs resolveRewrites(
|
|||
const std::vector<BuiltPathWithResult> & dependencies)
|
||||
{
|
||||
StringPairs res;
|
||||
for (auto & dep : dependencies)
|
||||
if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
|
||||
for (auto & [ outputName, outputPath ] : drvDep->outputs)
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
for (auto & dep : dependencies) {
|
||||
if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path)) {
|
||||
for (auto & [ outputName, outputPath ] : drvDep->outputs) {
|
||||
res.emplace(
|
||||
DownstreamPlaceholder::unknownCaOutput(
|
||||
drvDep->drvPath->outPath(), outputName).render(),
|
||||
DownstreamPlaceholder::fromSingleDerivedPathBuilt(
|
||||
SingleDerivedPath::Built {
|
||||
.drvPath = make_ref<SingleDerivedPath>(drvDep->drvPath->discardOutputPath()),
|
||||
.output = outputName,
|
||||
}).render(),
|
||||
store.printStorePath(outputPath)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
@ -6,4 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
|
|||
|
||||
clearStore
|
||||
|
||||
expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"
|
||||
out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
|
||||
|
||||
diff -r $out1 $out2
|
||||
|
|
|
@ -3,7 +3,8 @@ dyn-drv-tests := \
|
|||
$(d)/recursive-mod-json.sh \
|
||||
$(d)/build-built-drv.sh \
|
||||
$(d)/eval-outputOf.sh \
|
||||
$(d)/dep-built-drv.sh
|
||||
$(d)/dep-built-drv.sh \
|
||||
$(d)/old-daemon-error-hack.sh
|
||||
|
||||
install-tests-groups += dyn-drv
|
||||
|
||||
|
|
20
tests/dyn-drv/old-daemon-error-hack.nix
Normal file
20
tests/dyn-drv/old-daemon-error-hack.nix
Normal file
|
@ -0,0 +1,20 @@
|
|||
with import ./config.nix;
|
||||
|
||||
# A simple content-addressed derivation.
|
||||
# The derivation can be arbitrarily modified by passing a different `seed`,
|
||||
# but the output will always be the same
|
||||
rec {
|
||||
stub = mkDerivation {
|
||||
name = "stub";
|
||||
buildCommand = ''
|
||||
echo stub > $out
|
||||
'';
|
||||
};
|
||||
wrapper = mkDerivation {
|
||||
name = "has-dynamic-drv-dep";
|
||||
buildCommand = ''
|
||||
exit 1 # we're not building this derivation
|
||||
${builtins.outputOf stub.outPath "out"}
|
||||
'';
|
||||
};
|
||||
}
|
11
tests/dyn-drv/old-daemon-error-hack.sh
Normal file
11
tests/dyn-drv/old-daemon-error-hack.sh
Normal file
|
@ -0,0 +1,11 @@
|
|||
# Purposely bypassing our usual common for this subgroup
|
||||
source ../common.sh
|
||||
|
||||
# Need backend to support text-hashing too
|
||||
isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new"
|
||||
|
||||
enableFeatures "ca-derivations dynamic-derivations"
|
||||
|
||||
restartDaemon
|
||||
|
||||
expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations"
|
Loading…
Reference in a new issue