Merge pull request #8369 from obsidiansystems/inductive-derived-path

Make the Derived Path family of types inductive for dynamic derivations

doc/manual/src/release-notes/rl-next.md

@@ -16,3 +16,6 @@
   [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references)
   attribute is no longer guarded by an experimental flag and can be used
   freely.
+
+- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
+  This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.

src/build-remote/build-remote.cc

@@ -322,7 +322,12 @@ connected:
                 throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
         } else {
             copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
-            auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } });
+            auto res = sshStore->buildPathsWithResults({
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(*drvPath),
+                    .outputs = OutputsSpec::All {},
+                }
+            });
             // One path to build should produce exactly one build result
             assert(res.size() == 1);
             optResult = std::move(res[0]);

src/libcmd/built-path.cc

@@ -8,13 +8,39 @@
 
 namespace nix {
 
-nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
+#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
-    nlohmann::json res;
+    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
-    res["drvPath"] = store->printStorePath(drvPath);
+    { \
-    for (const auto& [output, path] : outputs) {
+        const MY_TYPE* me = this; \
-        res["outputs"][output] = store->printStorePath(path);
+        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        me = &other; \
+        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        return fields1 COMPARATOR fields2; \
     }
-    return res;
+#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+
+#define FIELD_TYPE std::pair<std::string, StorePath>
+CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
+#undef FIELD_TYPE
+
+#define FIELD_TYPE std::map<std::string, StorePath>
+CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
+#undef FIELD_TYPE
+
+#undef CMP
+#undef CMP_ONE
+
+StorePath SingleBuiltPath::outPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) { return p.path; },
+            [](const SingleBuiltPath::Built & b) { return b.output.second; },
+        }, raw()
+    );
 }
 
 StorePathSet BuiltPath::outPaths() const
@@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    for (const auto & [outputName, outputPath] : outputs) {
+        res["outputs"][outputName] = store.printStorePath(outputPath);
+    }
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    auto & [outputName, outputPath] = output;
+    res["output"] = outputName;
+    res["outputPath"] = store.printStorePath(outputPath);
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
+nlohmann::json BuiltPath::toJSON(const Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;
@@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
             [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
             [&](const BuiltPath::Built & p) {
                 auto drvHashes =
-                    staticOutputHashes(store, store.readDerivation(p.drvPath));
+                    staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
                 for (auto& [outputName, outputPath] : p.outputs) {
                     if (experimentalFeatureSettings.isEnabled(
                                 Xp::CaDerivations)) {
@@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
                         if (!drvOutput)
                             throw Error(
                                 "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
-                                store.printStorePath(p.drvPath), outputName);
+                                store.printStorePath(p.drvPath->outPath()), outputName);
                         auto thisRealisation = store.queryRealisation(
                             DrvOutput{*drvOutput, outputName});
                         assert(thisRealisation);  // We’ve built it, so we must

src/libcmd/built-path.hh

@@ -3,19 +3,60 @@
 
 namespace nix {
 
+struct SingleBuiltPath;
+
+struct SingleBuiltPathBuilt {
+    ref<SingleBuiltPath> drvPath;
+    std::pair<std::string, StorePath> output;
+
+    std::string to_string(const Store & store) const;
+    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
+
+    DECLARE_CMP(SingleBuiltPathBuilt);
+};
+
+using _SingleBuiltPathRaw = std::variant<
+    DerivedPathOpaque,
+    SingleBuiltPathBuilt
+>;
+
+struct SingleBuiltPath : _SingleBuiltPathRaw {
+    using Raw = _SingleBuiltPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = SingleBuiltPathBuilt;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    StorePath outPath() const;
+
+    static SingleBuiltPath parse(const Store & store, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
+};
+
+static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
+{
+    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
+}
+
 /**
  * A built derived path with hints in the form of optional concrete output paths.
  *
  * See 'BuiltPath' for more an explanation.
  */
 struct BuiltPathBuilt {
-    StorePath drvPath;
+    ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
-    nlohmann::json toJSON(ref<Store> store) const;
+    std::string to_string(const Store & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view);
+    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
 
-    GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs);
+    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -41,6 +82,7 @@ struct BuiltPath : _BuiltPathRaw {
     StorePathSet outPaths() const;
     RealisedPath::Set toRealisedPaths(Store & store) const;
 
+    nlohmann::json toJSON(const Store & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

src/libcmd/installable-attr-path.cc

@@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = drvPath,
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

src/libcmd/installable-derived-path.cc

@@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
 
 std::optional<StorePath> InstallableDerivedPath::getStorePath()
 {
-    return std::visit(overloaded {
+    return derivedPath.getBaseStorePath();
-        [&](const DerivedPath::Built & bfd) {
-            return bfd.drvPath;
-        },
-        [&](const DerivedPath::Opaque & bo) {
-            return bo.path;
-        },
-    }, derivedPath.raw());
 }
 
 InstallableDerivedPath InstallableDerivedPath::parse(
@@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
             // Remove this prior to stabilizing the new CLI.
             if (storePath.isDerivation()) {
                 auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = storePath,
+                    .drvPath = makeConstantStorePathRef(storePath),
                     .outputs = OutputsSpec::All { },
                 };
                 warn(
@@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         },
         // If the user did use ^, we just do exactly what is written.
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            drvRequireExperiment(*drv);
             return DerivedPath::Built {
-                .drvPath = store->parseStorePath(prefix),
+                .drvPath = std::move(drv),
                 .outputs = outputSpec,
             };
         },

src/libcmd/installable-flake.cc

@@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = std::move(drvPath),
+            .drvPath = makeConstantStorePathRef(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;

src/libcmd/installable-value.cc

@@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
 
     else if (v.type() == nString) {
         return {{
-            .path = state->coerceToDerivedPath(pos, v, errorCtx),
+            .path = DerivedPath::fromSingle(
+                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libcmd/installables.cc

@@ -515,6 +515,30 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
+static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
+{
+    return std::visit(
+        overloaded{
+            [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath {
+                return SingleBuiltPath::Opaque { bo.path };
+            },
+            [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath {
+                auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath);
+                // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work.
+                SingleDerivedPath::Built truncatedBfd {
+                    .drvPath = makeConstantStorePathRef(drvPath.outPath()),
+                    .output = bfd.output,
+                };
+                auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore);
+                return SingleBuiltPath::Built {
+                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
+                    .output = { bfd.output, outputPath },
+                };
+            },
+        },
+        b.raw());
+}
+
 std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
@@ -568,7 +592,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                     [&](const DerivedPath::Built & bfd) {
                         auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                             },
                             .info = aux.info}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
@@ -597,7 +624,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                         for (auto & [outputName, realisation] : buildResult.builtOutputs)
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                            },
                             .info = aux.info,
                             .result = buildResult}});
                     },
@@ -691,7 +721,7 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(bfd.drvPath);
+                    drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath));
                 },
             }, b.path.raw());
 

src/libcmd/repl.cc

@@ -648,7 +648,7 @@ bool NixRepl::processLine(std::string line)
         if (command == ":b" || command == ":bl") {
             state->store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = drvPath,
+                    .drvPath = makeConstantStorePathRef(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             });

src/libexpr/eval-cache.cc

@@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext()
                             return d.drvPath;
                         },
                         [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath;
+                            return b.drvPath->getBaseStorePath();
                         },
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;

src/libexpr/eval.cc

@@ -1042,7 +1042,7 @@ void EvalState::mkOutputString(
             : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
         NixStringContext {
             NixStringContextElem::Built {
-                .drvPath = drvPath,
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .output = outputName,
             }
         });
@@ -2299,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
 }
 
 
-std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
+std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
 {
     NixStringContext context;
     auto s = forceString(v, context, pos, errorCtx);
@@ -2310,21 +2310,16 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
             s, csize)
             .withTrace(pos, errorCtx).debugThrow<EvalError>();
     auto derivedPath = std::visit(overloaded {
-        [&](NixStringContextElem::Opaque && o) -> DerivedPath {
+        [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath {
-            return DerivedPath::Opaque {
+            return std::move(o);
-                .path = std::move(o.path),
-            };
         },
-        [&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
+        [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
             error(
                 "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
                 s).withTrace(pos, errorCtx).debugThrow<EvalError>();
         },
-        [&](NixStringContextElem::Built && b) -> DerivedPath {
+        [&](NixStringContextElem::Built && b) -> SingleDerivedPath {
-            return DerivedPath::Built {
+            return std::move(b);
-                .drvPath = std::move(b.drvPath),
-                .outputs = OutputsSpec::Names { std::move(b.output) },
-            };
         },
     }, ((NixStringContextElem &&) *context.begin()).raw());
     return {
@@ -2334,12 +2329,12 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
 }
 
 
-DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
+SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
 {
-    auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
+    auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
     auto s = s_;
     std::visit(overloaded {
-        [&](const DerivedPath::Opaque & o) {
+        [&](const SingleDerivedPath::Opaque & o) {
             auto sExpected = store->printStorePath(o.path);
             if (s != sExpected)
                 error(
@@ -2347,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str
                     s, sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
         },
-        [&](const DerivedPath::Built & b) {
+        [&](const SingleDerivedPath::Built & b) {
-            // TODO need derived path with single output to make this
+            auto sExpected = std::visit(overloaded {
-            // total. Will add as part of RFC 92 work and then this is
+                [&](const SingleDerivedPath::Opaque & o) {
-            // cleaned up.
+                    auto drv = store->readDerivation(o.path);
-            auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
+                    auto i = drv.outputs.find(b.output);
-
-            auto drv = store->readDerivation(b.drvPath);
-            auto i = drv.outputs.find(output);
                     if (i == drv.outputs.end())
-                throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
+                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
-            auto optOutputPath = i->second.path(*store, drv.name, output);
+                    auto optOutputPath = i->second.path(*store, drv.name, b.output);
                     // This is testing for the case of CA derivations
-            auto sExpected = optOutputPath
+                    return optOutputPath
                         ? store->printStorePath(*optOutputPath)
-                : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
+                        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
+                },
+                [&](const SingleDerivedPath::Built & o) {
+                    return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
+                },
+            }, b.drvPath->raw());
             if (s != sExpected)
                 error(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
-                    s, output, store->printStorePath(b.drvPath), sExpected)
+                    s, b.output, b.drvPath->to_string(*store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
         }
     }, derivedPath.raw());

src/libexpr/eval.hh

@@ -22,7 +22,7 @@ namespace nix {
 class Store;
 class EvalState;
 class StorePath;
-struct DerivedPath;
+struct SingleDerivedPath;
 enum RepairFlag : bool;
 
 
@@ -532,12 +532,12 @@ public:
     StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
 
     /**
-     * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
+     * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only.
      */
-    std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
+    std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
 
     /**
-     * Coerce to `DerivedPath`.
+     * Coerce to `SingleDerivedPath`.
      *
      * Must be a string which is either a literal store path or a
      * "placeholder (see `DownstreamPlaceholder`).
@@ -551,7 +551,7 @@ public:
      * source of truth, and ultimately tells us what we want, and then
      * we ensure the string corresponds to it.
      */
-    DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
+    SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
 public:
 

src/libexpr/primops.cc

@@ -56,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                     .drvPath = b.drvPath,
                     .outputs = OutputsSpec::Names { b.output },
                 });
-                ensureValid(b.drvPath);
+                ensureValid(b.drvPath->getBaseStorePath());
             },
             [&](const NixStringContextElem::Opaque & o) {
                 auto ctxS = store->printStorePath(o.path);
@@ -77,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     if (!evalSettings.enableImportFromDerivation)
         debugThrowLastTrace(Error(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
-            store->printStorePath(drvs.begin()->drvPath)));
+            drvs.begin()->to_string(*store)));
 
     /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
@@ -95,7 +95,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
                 res.insert_or_assign(
-                    DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(),
+                    DownstreamPlaceholder::fromSingleDerivedPathBuilt(
+                        SingleDerivedPath::Built {
+                            .drvPath = drv.drvPath,
+                            .output = outputName,
+                        }).render(),
                     store->printStorePath(outputPath)
                 );
             }
@@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v)
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                drv.inputDrvs[b.drvPath].insert(b.output);
+                if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
+                    drv.inputDrvs[p->path].insert(b.output);
+                else
+                    throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);

src/libexpr/primops/context.cc

@@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
                 contextInfos[std::move(d.drvPath)].allOutputs = true;
             },
             [&](NixStringContextElem::Built && b) {
-                contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output));
+                // FIXME should eventually show string context as is, no
+                // resolving here.
+                auto drvPath = resolveDerivedPath(*state.store, *b.drvPath);
+                contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output));
             },
             [&](NixStringContextElem::Opaque && o) {
                 contextInfos[std::move(o.path)].path = true;
@@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
             for (auto elem : iter->value->listItems()) {
                 auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
-                    .drvPath = namePath,
+                    .drvPath = makeConstantStorePathRef(namePath),
                     .output = std::string { outputName },
                 });
             }

src/libexpr/tests/derived-path.cc

@@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init)
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
     prop_opaque_path_round_trip,
-    (const DerivedPath::Opaque & o))
+    (const SingleDerivedPath::Opaque & o))
 {
     auto * v = state.allocValue();
     state.mkStorePathString(o.path, *v);
-    auto d = state.coerceToDerivedPath(noPos, *v, "");
+    auto d = state.coerceToSingleDerivedPath(noPos, *v, "");
-    RC_ASSERT(DerivedPath { o } == d);
+    RC_ASSERT(SingleDerivedPath { o } == d);
 }
 
 // TODO use DerivedPath::Built for parameter once it supports a single output
@@ -46,12 +46,12 @@ RC_GTEST_FIXTURE_PROP(
 
     auto * v = state.allocValue();
     state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings);
-    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    DerivedPath::Built b {
+    SingleDerivedPath::Built b {
-        .drvPath = drvPath,
+        .drvPath = makeConstantStorePathRef(drvPath),
-        .outputs = OutputsSpec::Names { outputName.name },
+        .output = outputName.name,
     };
-    RC_ASSERT(DerivedPath { b } == d);
+    RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
 RC_GTEST_FIXTURE_PROP(
@@ -61,12 +61,12 @@ RC_GTEST_FIXTURE_PROP(
 {
     auto * v = state.allocValue();
     state.mkOutputString(*v, drvPath, outputName.name, outPath);
-    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    DerivedPath::Built b {
+    SingleDerivedPath::Built b {
-        .drvPath = drvPath,
+        .drvPath = makeConstantStorePathRef(drvPath),
-        .outputs = OutputsSpec::Names { outputName.name },
+        .output = outputName.name,
     };
-    RC_ASSERT(DerivedPath { b } == d);
+    RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
 } /* namespace nix */

src/libexpr/tests/value/context.cc

@@ -8,6 +8,8 @@
 
 namespace nix {
 
+// Test a few cases of invalid string context elements.
+
 TEST(NixStringContextElemTest, empty_invalid) {
     EXPECT_THROW(
         NixStringContextElem::parse(""),
@@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) {
         BadStorePath);
 }
 
+/**
+ * Round trip (string <-> data structure) test for
+ * `NixStringContextElem::Opaque`.
+ */
 TEST(NixStringContextElemTest, opaque) {
     std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
     auto elem = NixStringContextElem::parse(opaque);
@@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) {
     ASSERT_EQ(elem.to_string(), opaque);
 }
 
+/**
+ * Round trip (string <-> data structure) test for
+ * `NixStringContextElem::DrvDeep`.
+ */
 TEST(NixStringContextElemTest, drvDeep) {
     std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
     auto elem = NixStringContextElem::parse(drvDeep);
@@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) {
     ASSERT_EQ(elem.to_string(), drvDeep);
 }
 
-TEST(NixStringContextElemTest, built) {
+/**
+ * Round trip (string <-> data structure) test for a simpler
+ * `NixStringContextElem::Built`.
+ */
+TEST(NixStringContextElemTest, built_opaque) {
     std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
     auto elem = NixStringContextElem::parse(built);
     auto * p = std::get_if<NixStringContextElem::Built>(&elem);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->output, "foo");
-    ASSERT_EQ(p->drvPath, StorePath { built.substr(5) });
+    ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
+        .path = StorePath { built.substr(5) },
+    }));
     ASSERT_EQ(elem.to_string(), built);
 }
 
+/**
+ * Round trip (string <-> data structure) test for a more complex,
+ * inductive `NixStringContextElem::Built`.
+ */
+TEST(NixStringContextElemTest, built_built) {
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+    mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+
+    std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
+    auto elem = NixStringContextElem::parse(built, mockXpSettings);
+    auto * p = std::get_if<NixStringContextElem::Built>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->output, "foo");
+    auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);
+    ASSERT_TRUE(drvPath);
+    ASSERT_EQ(drvPath->output, "bar");
+    ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
+        .path = StorePath { built.substr(9) },
+    }));
+    ASSERT_EQ(elem.to_string(), built);
+}
+
+/**
+ * Without the right experimental features enabled, we cannot parse a
+ * complex inductive string context element.
+ */
+TEST(NixStringContextElemTest, built_built_xp) {
+    ASSERT_THROW(
+        NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"),        MissingExperimentalFeature);
+}
+
 }
 
 namespace rc {
 using namespace nix;
 
-Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary()
-{
-    return gen::just(NixStringContextElem::Opaque {
-        .path = *gen::arbitrary<StorePath>(),
-    });
-}
-
 Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
 {
     return gen::just(NixStringContextElem::DrvDeep {
@@ -85,14 +129,6 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
     });
 }
 
-Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary()
-{
-    return gen::just(NixStringContextElem::Built {
-        .drvPath = *gen::arbitrary<StorePath>(),
-        .output = (*gen::arbitrary<StorePathName>()).name,
-    });
-}
-
 Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 {
     switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {

src/libexpr/value/context.cc

@@ -4,29 +4,52 @@
 
 namespace nix {
 
-NixStringContextElem NixStringContextElem::parse(std::string_view s0)
+NixStringContextElem NixStringContextElem::parse(
+    std::string_view s0,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     std::string_view s = s0;
 
+    std::function<SingleDerivedPath()> parseRest;
+    parseRest = [&]() -> SingleDerivedPath {
+        // Case on whether there is a '!'
+        size_t index = s.find("!");
+        if (index == std::string_view::npos) {
+            return SingleDerivedPath::Opaque {
+                .path = StorePath { s },
+            };
+        } else {
+            std::string output { s.substr(0, index) };
+            // Advance string to parse after the '!'
+            s = s.substr(index + 1);
+            auto drv = make_ref<SingleDerivedPath>(parseRest());
+            drvRequireExperiment(*drv, xpSettings);
+            return SingleDerivedPath::Built {
+                .drvPath = std::move(drv),
+                .output = std::move(output),
+            };
+        }
+    };
+
     if (s.size() == 0) {
         throw BadNixStringContextElem(s0,
             "String context element should never be an empty string");
     }
+
     switch (s.at(0)) {
     case '!': {
-        s = s.substr(1); // advance string to parse after first !
+        // Advance string to parse after the '!'
-        size_t index = s.find("!");
+        s = s.substr(1);
-        // This makes index + 1 safe. Index can be the length (one after index
+
-        // of last character), so given any valid character index --- a
+        // Find *second* '!'
-        // successful find --- we can add one.
+        if (s.find("!") == std::string_view::npos) {
-        if (index == std::string_view::npos) {
             throw BadNixStringContextElem(s0,
                 "String content element beginning with '!' should have a second '!'");
         }
-        return NixStringContextElem::Built {
+
-            .drvPath = StorePath { s.substr(index + 1) },
+        return std::visit(
-            .output = std::string(s.substr(0, index)),
+            [&](auto x) -> NixStringContextElem { return std::move(x); },
-        };
+            parseRest());
     }
     case '=': {
         return NixStringContextElem::DrvDeep {
@@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0)
         };
     }
     default: {
-        return NixStringContextElem::Opaque {
+        // Ensure no '!'
-            .path = StorePath { s },
+        if (s.find("!") != std::string_view::npos) {
-        };
+            throw BadNixStringContextElem(s0,
+                "String content element not beginning with '!' should not have a second '!'");
+        }
+        return std::visit(
+            [&](auto x) -> NixStringContextElem { return std::move(x); },
+            parseRest());
     }
     }
 }
 
-std::string NixStringContextElem::to_string() const {
+std::string NixStringContextElem::to_string() const
-    return std::visit(overloaded {
+{
-        [&](const NixStringContextElem::Built & b) {
     std::string res;
-            res += '!';
+
-            res += b.output;
+    std::function<void(const SingleDerivedPath &)> toStringRest;
-            res += '!';
+    toStringRest = [&](auto & p) {
-            res += b.drvPath.to_string();
+        std::visit(overloaded {
-            return res;
+            [&](const SingleDerivedPath::Opaque & o) {
+                res += o.path.to_string();
             },
-        [&](const NixStringContextElem::DrvDeep & d) {
+            [&](const SingleDerivedPath::Built & o) {
-            std::string res;
+                res += o.output;
-            res += '=';
+                res += '!';
-            res += d.drvPath.to_string();
+                toStringRest(*o.drvPath);
-            return res;
+            },
+        }, p.raw());
+    };
+
+    std::visit(overloaded {
+        [&](const NixStringContextElem::Built & b) {
+            res += '!';
+            toStringRest(b);
         },
         [&](const NixStringContextElem::Opaque & o) {
-            return std::string { o.path.to_string() };
+            toStringRest(o);
+        },
+        [&](const NixStringContextElem::DrvDeep & d) {
+            res += '=';
+            res += d.drvPath.to_string();
         },
     }, raw());
+
+    return res;
 }
 
 }

src/libexpr/value/context.hh

@@ -3,7 +3,7 @@
 
 #include "util.hh"
 #include "comparator.hh"
-#include "path.hh"
+#include "derived-path.hh"
 
 #include <variant>
 
@@ -31,11 +31,7 @@ public:
  *
  * Encoded as just the path: ‘<path>’.
  */
-struct NixStringContextElem_Opaque {
+typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque;
-    StorePath path;
-
-    GENERATE_CMP(NixStringContextElem_Opaque, me->path);
-};
 
 /**
  * Path to a derivation and its entire build closure.
@@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep {
  *
  * Encoded in the form ‘!<output>!<drvPath>’.
  */
-struct NixStringContextElem_Built {
+typedef SingleDerivedPath::Built NixStringContextElem_Built;
-    StorePath drvPath;
-    std::string output;
-
-    GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output);
-};
 
 using _NixStringContextElem_Raw = std::variant<
     NixStringContextElem_Opaque,
@@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
      * - ‘<path>’
      * - ‘=<path>’
      * - ‘!<name>!<path>’
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static NixStringContextElem parse(std::string_view s);
+    static NixStringContextElem parse(
+        std::string_view s,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     std::string to_string() const;
 };
 

src/libstore/build/derivation-goal.cc

@@ -65,7 +65,7 @@ namespace nix {
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
+    : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
     , useDerivation(true)
     , drvPath(drvPath)
     , wantedOutputs(wantedOutputs)
@@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
-        DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
     const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
+    : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
     , useDerivation(false)
     , drvPath(drvPath)
     , wantedOutputs(wantedOutputs)
@@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
-        DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
 
     for (auto & outputName : outputs->second) {
         auto buildResult = dg->getBuildResult(DerivedPath::Built {
-            .drvPath = dg->drvPath,
+            .drvPath = makeConstantStorePathRef(dg->drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {

src/libstore/build/entry-points.cc

@@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     try {
         worker.run(Goals{goal});
         return goal->getBuildResult(DerivedPath::Built {
-            .drvPath = drvPath,
+            .drvPath = makeConstantStorePathRef(drvPath),
             .outputs = OutputsSpec::All {},
         });
     } catch (Error & e) {

src/libstore/build/local-derivation-goal.cc

@@ -1172,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs()
 }
 
 
+static StorePath pathPartOfReq(const SingleDerivedPath & req)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+        [&](const SingleDerivedPath::Built & bfd)  {
+            return pathPartOfReq(*bfd.drvPath);
+        },
+    }, req.raw());
+}
+
+
 static StorePath pathPartOfReq(const DerivedPath & req)
 {
     return std::visit(overloaded {
@@ -1179,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req)
             return bo.path;
         },
         [&](const DerivedPath::Built & bfd)  {
-            return bfd.drvPath;
+            return pathPartOfReq(*bfd.drvPath);
         },
     }, req.raw());
 }

src/libstore/build/worker.cc

@@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & bfd) -> GoalPtr {
-            return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
+            if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
+                return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
+            else
+                throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
         [&](const DerivedPath::Opaque & bo) -> GoalPtr {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals)
     for (auto & i : _topGoals) {
         topGoals.insert(i);
         if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
-            topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
+            topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs});
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Opaque{goal->storePath});
         }

src/libstore/derived-path.cc

@@ -7,52 +7,133 @@
 
 namespace nix {
 
-nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
+#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
+    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
+    { \
+        const MY_TYPE* me = this; \
+        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        me = &other; \
+        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        return fields1 COMPARATOR fields2; \
+    }
+#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+
+#define FIELD_TYPE std::string
+CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
+#undef FIELD_TYPE
+
+#define FIELD_TYPE OutputsSpec
+CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
+#undef FIELD_TYPE
+
+#undef CMP
+#undef CMP_ONE
+
+nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
+{
+    return store.printStorePath(path);
+}
+
+nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const {
     nlohmann::json res;
-    res["path"] = store->printStorePath(path);
+    res["drvPath"] = drvPath->toJSON(store);
+    // Fallback for the input-addressed derivation case: We expect to always be
+    // able to print the output paths, so let’s do it
+    // FIXME try-resolve on drvPath
+    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
+    res["output"] = output;
+    auto outputPathIter = outputMap.find(output);
+    if (outputPathIter == outputMap.end())
+        res["outputPath"] = nullptr;
+    else if (std::optional p = outputPathIter->second)
+        res["outputPath"] = store.printStorePath(*p);
+    else
+        res["outputPath"] = nullptr;
     return res;
 }
 
-nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
+nlohmann::json DerivedPath::Built::toJSON(Store & store) const {
     nlohmann::json res;
-    res["drvPath"] = store->printStorePath(drvPath);
+    res["drvPath"] = drvPath->toJSON(store);
     // Fallback for the input-addressed derivation case: We expect to always be
     // able to print the output paths, so let’s do it
-    const auto outputMap = store->queryPartialDerivationOutputMap(drvPath);
+    // FIXME try-resolve on drvPath
+    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
     for (const auto & [output, outputPathOpt] : outputMap) {
         if (!outputs.contains(output)) continue;
         if (outputPathOpt)
-            res["outputs"][output] = store->printStorePath(*outputPathOpt);
+            res["outputs"][output] = store.printStorePath(*outputPathOpt);
         else
             res["outputs"][output] = nullptr;
     }
     return res;
 }
 
+nlohmann::json SingleDerivedPath::toJSON(Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
+nlohmann::json DerivedPath::toJSON(Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
 std::string DerivedPath::Opaque::to_string(const Store & store) const
 {
     return store.printStorePath(path);
 }
 
+std::string SingleDerivedPath::Built::to_string(const Store & store) const
+{
+    return drvPath->to_string(store) + "^" + output;
+}
+
+std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
+{
+    return drvPath->to_string(store) + "!" + output;
+}
+
 std::string DerivedPath::Built::to_string(const Store & store) const
 {
-    return store.printStorePath(drvPath)
+    return drvPath->to_string(store)
         + '^'
         + outputs.to_string();
 }
 
 std::string DerivedPath::Built::to_string_legacy(const Store & store) const
 {
-    return store.printStorePath(drvPath)
+    return drvPath->to_string_legacy(store)
-        + '!'
+        + "!"
         + outputs.to_string();
 }
 
+std::string SingleDerivedPath::to_string(const Store & store) const
+{
+    return std::visit(
+        [&](const auto & req) { return req.to_string(store); },
+        raw());
+}
+
 std::string DerivedPath::to_string(const Store & store) const
+{
+    return std::visit(
+        [&](const auto & req) { return req.to_string(store); },
+        raw());
+}
+
+std::string SingleDerivedPath::to_string_legacy(const Store & store) const
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & req) { return req.to_string(store); },
+        [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
-        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+        [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); },
     }, this->raw());
 }
 
@@ -70,30 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
     return {store.parseStorePath(s)};
 }
 
-DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
+void drvRequireExperiment(
+    const SingleDerivedPath & drv,
+    const ExperimentalFeatureSettings & xpSettings)
 {
+    std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque &) {
+            // plain drv path; no experimental features required.
+        },
+        [&](const SingleDerivedPath::Built &) {
+            xpSettings.require(Xp::DynamicDerivations);
+        },
+    }, drv.raw());
+}
+
+SingleDerivedPath::Built SingleDerivedPath::Built::parse(
+    const Store & store, ref<SingleDerivedPath> drv,
+    std::string_view output,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    drvRequireExperiment(*drv, xpSettings);
     return {
-        .drvPath = store.parseStorePath(drvS),
+        .drvPath = drv,
+        .output = std::string { output },
+    };
+}
+
+DerivedPath::Built DerivedPath::Built::parse(
+    const Store & store, ref<SingleDerivedPath> drv,
+    std::string_view outputsS,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    drvRequireExperiment(*drv, xpSettings);
+    return {
+        .drvPath = drv,
         .outputs = OutputsSpec::parse(outputsS),
     };
 }
 
-static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
+static SingleDerivedPath parseWithSingle(
+    const Store & store, std::string_view s, std::string_view separator,
+    const ExperimentalFeatureSettings & xpSettings)
 {
-    size_t n = s.find(separator);
+    size_t n = s.rfind(separator);
+    return n == s.npos
+        ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s)
+        : (SingleDerivedPath) SingleDerivedPath::Built::parse(store,
+            make_ref<SingleDerivedPath>(parseWithSingle(
+                store,
+                s.substr(0, n),
+                separator,
+                xpSettings)),
+            s.substr(n + 1),
+            xpSettings);
+}
+
+SingleDerivedPath SingleDerivedPath::parse(
+    const Store & store,
+    std::string_view s,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    return parseWithSingle(store, s, "^", xpSettings);
+}
+
+SingleDerivedPath SingleDerivedPath::parseLegacy(
+    const Store & store,
+    std::string_view s,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    return parseWithSingle(store, s, "!", xpSettings);
+}
+
+static DerivedPath parseWith(
+    const Store & store, std::string_view s, std::string_view separator,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    size_t n = s.rfind(separator);
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
-        : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
+        : (DerivedPath) DerivedPath::Built::parse(store,
+            make_ref<SingleDerivedPath>(parseWithSingle(
+                store,
+                s.substr(0, n),
+                separator,
+                xpSettings)),
+            s.substr(n + 1),
+            xpSettings);
 }
 
-DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+DerivedPath DerivedPath::parse(
+    const Store & store,
+    std::string_view s,
+    const ExperimentalFeatureSettings & xpSettings)
 {
-	return parseWith(store, s, "^");
+    return parseWith(store, s, "^", xpSettings);
 }
 
-DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
+DerivedPath DerivedPath::parseLegacy(
+    const Store & store,
+    std::string_view s,
+    const ExperimentalFeatureSettings & xpSettings)
 {
-	return parseWith(store, s, "!");
+    return parseWith(store, s, "!", xpSettings);
+}
+
+DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & o) -> DerivedPath {
+            return o;
+        },
+        [&](const SingleDerivedPath::Built & b) -> DerivedPath {
+            return DerivedPath::Built {
+                .drvPath = b.drvPath,
+                .outputs = OutputsSpec::Names { b.output },
+            };
+        },
+    }, req.raw());
+}
+
+const StorePath & SingleDerivedPath::Built::getBaseStorePath() const
+{
+	return drvPath->getBaseStorePath();
+}
+
+const StorePath & DerivedPath::Built::getBaseStorePath() const
+{
+	return drvPath->getBaseStorePath();
+}
+
+template<typename DP>
+static inline const StorePath & getBaseStorePath_(const DP & derivedPath)
+{
+    return std::visit(overloaded {
+        [&](const typename DP::Built & bfd) -> auto & {
+            return bfd.drvPath->getBaseStorePath();
+        },
+        [&](const typename DP::Opaque & bo) -> auto & {
+            return bo.path;
+        },
+    }, derivedPath.raw());
+}
+
+const StorePath & SingleDerivedPath::getBaseStorePath() const
+{
+	return getBaseStorePath_(*this);
+}
+
+const StorePath & DerivedPath::getBaseStorePath() const
+{
+	return getBaseStorePath_(*this);
 }
 
 }

src/libstore/derived-path.hh

@@ -24,28 +24,37 @@ class Store;
 struct DerivedPathOpaque {
     StorePath path;
 
-    nlohmann::json toJSON(ref<Store> store) const;
     std::string to_string(const Store & store) const;
     static DerivedPathOpaque parse(const Store & store, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
 
     GENERATE_CMP(DerivedPathOpaque, me->path);
 };
 
+struct SingleDerivedPath;
+
 /**
- * A derived path that is built from a derivation
+ * A single derived path that is built from a derivation
  *
- * Built derived paths are pair of a derivation and some output names.
+ * Built derived paths are pair of a derivation and an output name. They are
- * They are evaluated by building the derivation, and then replacing the
+ * evaluated by building the derivation, and then taking the resulting output
- * output names with the resulting outputs.
+ * path of the given output name.
- *
- * Note that does mean a derived store paths evaluates to multiple
- * opaque paths, which is sort of icky as expressions are supposed to
- * evaluate to single values. Perhaps this should have just a single
- * output name.
  */
-struct DerivedPathBuilt {
+struct SingleDerivedPathBuilt {
-    StorePath drvPath;
+    ref<SingleDerivedPath> drvPath;
-    OutputsSpec outputs;
+    std::string output;
+
+    /**
+     * Get the store path this is ultimately derived from (by realising
+     * and projecting outputs).
+     *
+     * Note that this is *not* a property of the store object being
+     * referred to, but just of this path --- how we happened to be
+     * referring to that store object. In other words, this means this
+     * function breaks "referential transparency". It should therefore
+     * be used only with great care.
+     */
+    const StorePath & getBaseStorePath() const;
 
     /**
      * Uses `^` as the separator
@@ -57,11 +66,139 @@ struct DerivedPathBuilt {
     std::string to_string_legacy(const Store & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
+    static SingleDerivedPathBuilt parse(
-    nlohmann::json toJSON(ref<Store> store) const;
+        const Store & store, ref<SingleDerivedPath> drvPath,
+        std::string_view outputs,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+    nlohmann::json toJSON(Store & store) const;
 
-    GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
+    DECLARE_CMP(SingleDerivedPathBuilt);
+};
+
+using _SingleDerivedPathRaw = std::variant<
+    DerivedPathOpaque,
+    SingleDerivedPathBuilt
+>;
+
+/**
+ * A "derived path" is a very simple sort of expression (not a Nix
+ * language expression! But an expression in a the general sense) that
+ * evaluates to (concrete) store path. It is either:
+ *
+ * - opaque, in which case it is just a concrete store path with
+ *   possibly no known derivation
+ *
+ * - built, in which case it is a pair of a derivation path and an
+ *   output name.
+ */
+struct SingleDerivedPath : _SingleDerivedPathRaw {
+    using Raw = _SingleDerivedPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = SingleDerivedPathBuilt;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    /**
+     * Get the store path this is ultimately derived from (by realising
+     * and projecting outputs).
+     *
+     * Note that this is *not* a property of the store object being
+     * referred to, but just of this path --- how we happened to be
+     * referring to that store object. In other words, this means this
+     * function breaks "referential transparency". It should therefore
+     * be used only with great care.
+     */
+    const StorePath & getBaseStorePath() const;
+
+    /**
+     * Uses `^` as the separator
+     */
+    std::string to_string(const Store & store) const;
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * Uses `^` as the separator
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
+    static SingleDerivedPath parse(
+        const Store & store,
+        std::string_view,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+    /**
+     * Uses `!` as the separator
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
+    static SingleDerivedPath parseLegacy(
+        const Store & store,
+        std::string_view,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+    nlohmann::json toJSON(Store & store) const;
+};
+
+static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath)
+{
+    return make_ref<SingleDerivedPath>(SingleDerivedPath::Opaque { drvPath });
+}
+
+/**
+ * A set of derived paths that are built from a derivation
+ *
+ * Built derived paths are pair of a derivation and some output names.
+ * They are evaluated by building the derivation, and then replacing the
+ * output names with the resulting outputs.
+ *
+ * Note that does mean a derived store paths evaluates to multiple
+ * opaque paths, which is sort of icky as expressions are supposed to
+ * evaluate to single values. Perhaps this should have just a single
+ * output name.
+ */
+struct DerivedPathBuilt {
+    ref<SingleDerivedPath> drvPath;
+    OutputsSpec outputs;
+
+    /**
+     * Get the store path this is ultimately derived from (by realising
+     * and projecting outputs).
+     *
+     * Note that this is *not* a property of the store object being
+     * referred to, but just of this path --- how we happened to be
+     * referring to that store object. In other words, this means this
+     * function breaks "referential transparency". It should therefore
+     * be used only with great care.
+     */
+    const StorePath & getBaseStorePath() const;
+
+    /**
+     * Uses `^` as the separator
+     */
+    std::string to_string(const Store & store) const;
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * The caller splits on the separator, so it works for both variants.
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
+    static DerivedPathBuilt parse(
+        const Store & store, ref<SingleDerivedPath>,
+        std::string_view,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+    nlohmann::json toJSON(Store & store) const;
+
+    DECLARE_CMP(DerivedPathBuilt);
 };
 
 using _DerivedPathRaw = std::variant<
@@ -71,13 +208,13 @@ using _DerivedPathRaw = std::variant<
 
 /**
  * A "derived path" is a very simple sort of expression that evaluates
- * to (concrete) store path. It is either:
+ * to one or more (concrete) store paths. It is either:
  *
- * - opaque, in which case it is just a concrete store path with
+ * - opaque, in which case it is just a single concrete store path with
  *   possibly no known derivation
  *
- * - built, in which case it is a pair of a derivation path and an
+ * - built, in which case it is a pair of a derivation path and some
- *   output name.
+ *   output names.
  */
 struct DerivedPath : _DerivedPathRaw {
     using Raw = _DerivedPathRaw;
@@ -90,6 +227,18 @@ struct DerivedPath : _DerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    /**
+     * Get the store path this is ultimately derived from (by realising
+     * and projecting outputs).
+     *
+     * Note that this is *not* a property of the store object being
+     * referred to, but just of this path --- how we happened to be
+     * referring to that store object. In other words, this means this
+     * function breaks "referential transparency". It should therefore
+     * be used only with great care.
+     */
+    const StorePath & getBaseStorePath() const;
+
     /**
      * Uses `^` as the separator
      */
@@ -100,14 +249,43 @@ struct DerivedPath : _DerivedPathRaw {
     std::string to_string_legacy(const Store & store) const;
     /**
      * Uses `^` as the separator
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static DerivedPath parse(const Store & store, std::string_view);
+    static DerivedPath parse(
+        const Store & store,
+        std::string_view,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
      * Uses `!` as the separator
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static DerivedPath parseLegacy(const Store & store, std::string_view);
+    static DerivedPath parseLegacy(
+        const Store & store,
+        std::string_view,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+    /**
+     * Convert a `SingleDerivedPath` to a `DerivedPath`.
+     */
+    static DerivedPath fromSingle(const SingleDerivedPath &);
+
+    nlohmann::json toJSON(Store & store) const;
 };
 
 typedef std::vector<DerivedPath> DerivedPaths;
 
+/**
+ * Used by various parser functions to require experimental features as
+ * needed.
+ *
+ * Somewhat unfortunate this cannot just be an implementation detail for
+ * this module.
+ *
+ * @param xpSettings Stop-gap to avoid globals during unit tests.
+ */
+void drvRequireExperiment(
+    const SingleDerivedPath & drv,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 }

src/libstore/downstream-placeholder.cc

@@ -38,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
     };
 }
 
+DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
+    const SingleDerivedPath::Built & b)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & o) {
+            return DownstreamPlaceholder::unknownCaOutput(o.path, b.output);
+        },
+        [&](const SingleDerivedPath::Built & b2) {
+            return DownstreamPlaceholder::unknownDerivation(
+                DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2),
+                b.output);
+        },
+    }, b.drvPath->raw());
+}
+
 }

src/libstore/downstream-placeholder.hh

@@ -3,6 +3,7 @@
 
 #include "hash.hh"
 #include "path.hh"
+#include "derived-path.hh"
 
 namespace nix {
 
@@ -73,6 +74,17 @@ public:
         const DownstreamPlaceholder & drvPlaceholder,
         std::string_view outputName,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+    /**
+     * Convenience constructor that handles both cases (unknown
+     * content-addressed output and unknown derivation), delegating as
+     * needed to `unknownCaOutput` and `unknownDerivation`.
+     *
+     * Recursively builds up a placeholder from a
+     * `SingleDerivedPath::Built.drvPath` chain.
+     */
+    static DownstreamPlaceholder fromSingleDerivedPathBuilt(
+        const SingleDerivedPath::Built & built);
 };
 
 }

src/libstore/legacy-ssh-store.cc

@@ -358,6 +358,9 @@ public:
                 [&](const StorePath & drvPath) {
                     throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
                 },
+                [&](std::monostate) {
+                    throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
+                },
             }, sOrDrvPath);
         }
         conn->to << ss;

src/libstore/misc.cc

@@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
         }
 
         for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second }));
+            pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second }));
     };
 
     auto checkOutput = [&](
@@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
         std::visit(overloaded {
           [&](const DerivedPath::Built & bfd) {
-            if (!isValidPath(bfd.drvPath)) {
+            auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath);
+            if (!drvPathP) {
+                // TODO make work in this case.
+                warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this));
+                return;
+            }
+            auto & drvPath = drvPathP->path;
+
+            if (!isValidPath(drvPath)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
-                state->unknown.insert(bfd.drvPath);
+                state->unknown.insert(drvPath);
                 return;
             }
 
@@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
             /* true for regular derivations, and CA derivations for which we
                have a trust mapping for all wanted outputs. */
             auto knownOutputPaths = true;
-            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
+            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) {
                 if (!pathOpt) {
                     knownOutputPaths = false;
                     break;
@@ -197,8 +205,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
             }
             if (knownOutputPaths && invalid.empty()) return;
 
-            auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
+            auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
-            ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
+            ParsedDerivation parsedDrv(StorePath(drvPath), *drv);
 
             if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -233,9 +241,9 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
             if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
-                    pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
+                    pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState));
             } else
-                mustBuildDrv(bfd.drvPath, *drv);
+                mustBuildDrv(drvPath, *drv);
 
           },
           [&](const DerivedPath::Opaque & bo) {
@@ -340,7 +348,9 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 
 OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
 {
-    auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_);
+    auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
+
+    auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_);
 
     auto outputsOpt = std::visit(overloaded {
         [&](const OutputsSpec::All &) {
@@ -355,7 +365,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
                 if (!pOutputPathOpt)
                     throw Error(
                         "the derivation '%s' doesn't have an output named '%s'",
-                        store.printStorePath(bfd.drvPath), output);
+                        bfd.drvPath->to_string(store), output);
                 outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt));
             }
             return outputsOpt;
@@ -365,11 +375,64 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
     OutputPathMap outputs;
     for (auto & [outputName, outputPathOpt] : outputsOpt) {
         if (!outputPathOpt)
-            throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName);
+            throw MissingRealisation(bfd.drvPath->to_string(store), outputName);
         auto & outputPath = *outputPathOpt;
         outputs.insert_or_assign(outputName, outputPath);
     }
     return outputs;
 }
 
+
+StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_)
+{
+    auto & evalStore = evalStore_ ? *evalStore_ : store;
+
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+        [&](const SingleDerivedPath::Built & bfd) {
+            auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
+            auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_);
+            if (outputPaths.count(bfd.output) == 0)
+                throw Error("derivation '%s' does not have an output named '%s'",
+                    store.printStorePath(drvPath), bfd.output);
+            auto & optPath = outputPaths.at(bfd.output);
+            if (!optPath)
+                throw Error("'%s' does not yet map to a known concrete store path",
+                    bfd.to_string(store));
+            return *optPath;
+        },
+    }, req.raw());
+}
+
+
+OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd)
+{
+    auto drvPath = resolveDerivedPath(store, *bfd.drvPath);
+    auto outputMap = store.queryDerivationOutputMap(drvPath);
+    auto outputsLeft = std::visit(overloaded {
+        [&](const OutputsSpec::All &) {
+            return StringSet {};
+        },
+        [&](const OutputsSpec::Names & names) {
+            return static_cast<StringSet>(names);
+        },
+    }, bfd.outputs.raw());
+    for (auto iter = outputMap.begin(); iter != outputMap.end();) {
+        auto & outputName = iter->first;
+        if (bfd.outputs.contains(outputName)) {
+            outputsLeft.erase(outputName);
+            ++iter;
+        } else {
+            iter = outputMap.erase(iter);
+        }
+    }
+    if (!outputsLeft.empty())
+        throw Error("derivation '%s' does not have an outputs %s",
+            store.printStorePath(drvPath),
+            concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs))));
+    return outputMap;
+}
+
 }

src/libstore/path-with-outputs.cc

@@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
 DerivedPath StorePathWithOutputs::toDerivedPath() const
 {
     if (!outputs.empty()) {
-        return DerivedPath::Built { path, OutputsSpec::Names { outputs } };
+        return DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(path),
+            .outputs = OutputsSpec::Names { outputs },
+        };
     } else if (path.isDerivation()) {
         assert(outputs.empty());
-        return DerivedPath::Built { path, OutputsSpec::All { } };
+        return DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(path),
+            .outputs = OutputsSpec::All { },
+        };
     } else {
         return DerivedPath::Opaque { path };
     }
@@ -34,19 +40,21 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>
 }
 
 
-std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
+StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult {
             if (bo.path.isDerivation()) {
                 // drv path gets interpreted as "build", not "get drv file itself"
                 return bo.path;
             }
             return StorePathWithOutputs { bo.path };
         },
-        [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult {
+            return std::visit(overloaded {
+                [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult {
                     return StorePathWithOutputs {
-                .path = bfd.drvPath,
+                        .path = bo.path,
                         // Use legacy encoding of wildcard as empty set
                         .outputs = std::visit(overloaded {
                             [&](const OutputsSpec::All &) -> StringSet {
@@ -58,6 +66,11 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv
                         }, bfd.outputs.raw()),
                     };
                 },
+                [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult {
+                    return std::monostate {};
+                },
+            }, bfd.drvPath->raw());
+        },
     }, p.raw());
 }
 

src/libstore/path-with-outputs.hh

@@ -23,7 +23,9 @@ struct StorePathWithOutputs
 
     DerivedPath toDerivedPath() const;
 
-    static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &);
+    typedef std::variant<StorePathWithOutputs, StorePath, std::monostate> ParseResult;
+
+    static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &);
 };
 
 std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>);

src/libstore/remote-store.cc

@@ -656,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con
                         GET_PROTOCOL_MAJOR(conn.daemonVersion),
                         GET_PROTOCOL_MINOR(conn.daemonVersion));
                 },
+                [&](std::monostate) {
+                    throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'");
+                },
             }, sOrDrvPath);
         }
         conn.to << ss;
@@ -670,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore(
         /* The remote doesn't have a way to access evalStore, so copy
            the .drvs. */
         RealisedPath::Set drvPaths2;
-        for (auto & i : paths)
+        for (const auto & i : paths) {
-            if (auto p = std::get_if<DerivedPath::Built>(&i))
+            std::visit(overloaded {
-                drvPaths2.insert(p->drvPath);
+                [&](const DerivedPath::Opaque & bp) {
+                    // Do nothing, path is hopefully there already
+                },
+                [&](const DerivedPath::Built & bp) {
+                    drvPaths2.insert(bp.drvPath->getBaseStorePath());
+                },
+            }, i.raw());
+        }
         copyClosure(*evalStore, *this, drvPaths2);
     }
 }
@@ -742,7 +752,8 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
                         };
 
                         OutputPathMap outputs;
-                        auto drv = evalStore->readDerivation(bfd.drvPath);
+                        auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
+                        auto drv = evalStore->readDerivation(drvPath);
                         const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
                         auto built = resolveDerivedPath(*this, bfd, &*evalStore);
                         for (auto & [output, outputPath] : built) {
@@ -750,7 +761,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
                             if (!outputHash)
                                 throw Error(
                                     "the derivation '%s' doesn't have an output named '%s'",
-                                    printStorePath(bfd.drvPath), output);
+                                    printStorePath(drvPath), output);
                             auto outputId = DrvOutput{ *outputHash, output };
                             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
                                 auto realisation =

src/libstore/store-api.hh

@@ -945,6 +945,7 @@ void removeTempRoots();
  * Resolve the derived path completely, failing if any derivation output
  * is unknown.
  */
+StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr);
 OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);
 
 

src/libstore/tests/derived-path.cc

@@ -17,14 +17,34 @@ Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
     });
 }
 
+Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
+{
+    return gen::just(SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
+        .output = (*gen::arbitrary<StorePathName>()).name,
+    });
+}
+
 Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
 {
     return gen::just(DerivedPath::Built {
-        .drvPath = *gen::arbitrary<StorePath>(),
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
         .outputs = *gen::arbitrary<OutputsSpec>(),
     });
 }
 
+Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
+    case 0:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
+    case 1:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
+    default:
+        assert(false);
+    }
+}
+
 Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
 {
     switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
@@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest
 {
 };
 
-// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
+/**
-// no a real fixture.
+ * Round trip (string <-> data structure) test for
-//
+ * `DerivedPath::Opaque`.
-// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
+ */
-TEST_F(DerivedPathTest, force_init)
+TEST_F(DerivedPathTest, opaque) {
-{
+    std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
+    auto elem = DerivedPath::parse(*store, opaque);
+    auto * p = std::get_if<DerivedPath::Opaque>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->path, store->parseStorePath(opaque));
+    ASSERT_EQ(elem.to_string(*store), opaque);
+}
+
+/**
+ * Round trip (string <-> data structure) test for a simpler
+ * `DerivedPath::Built`.
+ */
+TEST_F(DerivedPathTest, built_opaque) {
+    std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo";
+    auto elem = DerivedPath::parse(*store, built);
+    auto * p = std::get_if<DerivedPath::Built>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" }));
+    ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
+        .path = store->parseStorePath(built.substr(0, 49)),
+    }));
+    ASSERT_EQ(elem.to_string(*store), built);
+}
+
+/**
+ * Round trip (string <-> data structure) test for a more complex,
+ * inductive `DerivedPath::Built`.
+ */
+TEST_F(DerivedPathTest, built_built) {
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+    mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+
+    std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz";
+    auto elem = DerivedPath::parse(*store, built, mockXpSettings);
+    auto * p = std::get_if<DerivedPath::Built>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" }));
+    auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);
+    ASSERT_TRUE(drvPath);
+    ASSERT_EQ(drvPath->output, "foo");
+    ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
+        .path = store->parseStorePath(built.substr(0, 49)),
+    }));
+    ASSERT_EQ(elem.to_string(*store), built);
+}
+
+/**
+ * Without the right experimental features enabled, we cannot parse a
+ * complex inductive derived path.
+ */
+TEST_F(DerivedPathTest, built_built_xp) {
+    ASSERT_THROW(
+        DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"),
+        MissingExperimentalFeature);
 }
 
 RC_GTEST_FIXTURE_PROP(

src/libstore/tests/derived-path.hh

@@ -12,8 +12,18 @@ namespace rc {
 using namespace nix;
 
 template<>
-struct Arbitrary<DerivedPath::Opaque> {
+struct Arbitrary<SingleDerivedPath::Opaque> {
-    static Gen<DerivedPath::Opaque> arbitrary();
+    static Gen<SingleDerivedPath::Opaque> arbitrary();
+};
+
+template<>
+struct Arbitrary<SingleDerivedPath::Built> {
+    static Gen<SingleDerivedPath::Built> arbitrary();
+};
+
+template<>
+struct Arbitrary<SingleDerivedPath> {
+    static Gen<SingleDerivedPath> arbitrary();
 };
 
 template<>

src/libutil/comparator.hh

@@ -1,6 +1,22 @@
 #pragma once
 ///@file
 
+/**
+ * Declare comparison methods without defining them.
+ */
+#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \
+    bool operator COMPARATOR(const MY_TYPE & other) const;
+#define DECLARE_EQUAL(my_type) \
+    DECLARE_ONE_CMP(==, my_type)
+#define DECLARE_LEQ(my_type) \
+    DECLARE_ONE_CMP(<, my_type)
+#define DECLARE_NEQ(my_type) \
+    DECLARE_ONE_CMP(!=, my_type)
+#define DECLARE_CMP(my_type) \
+    DECLARE_EQUAL(my_type) \
+    DECLARE_LEQ(my_type) \
+    DECLARE_NEQ(my_type)
+
 /**
  * Awful hacky generation of the comparison operators by doing a lexicographic
  * comparison between the choosen fields.

src/nix-build/nix-build.cc

@@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv)
 
                 auto bashDrv = drv->requireDrvPath();
                 pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = bashDrv,
+                    .drvPath = makeConstantStorePathRef(bashDrv),
                     .outputs = OutputsSpec::Names {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
@@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv)
                     }))
             {
                 pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = inputDrv,
+                    .drvPath = makeConstantStorePathRef(inputDrv),
                     .outputs = OutputsSpec::Names { inputOutputs },
                 });
                 pathsToCopy.insert(inputDrv);
@@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv)
             if (outputName == "")
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
-            pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
+            pathsToBuild.push_back(DerivedPath::Built{
+                .drvPath = makeConstantStorePathRef(drvPath),
+                .outputs = OutputsSpec::Names{outputName},
+            });
             pathsToBuildOrdered.push_back({drvPath, {outputName}});
             drvsToCopy.insert(drvPath);
 

src/nix-env/nix-env.cc

@@ -481,7 +481,7 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath())
             targets.push_back(DerivedPath::Built{
-                .drvPath = *drvPath,
+                .drvPath = makeConstantStorePathRef(*drvPath),
                 .outputs = OutputsSpec::All { },
             });
         else
@@ -759,7 +759,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     std::vector<DerivedPath> paths {
         drvPath
         ? (DerivedPath) (DerivedPath::Built {
-            .drvPath = *drvPath,
+            .drvPath = makeConstantStorePathRef(*drvPath),
             .outputs = OutputsSpec::All { },
         })
         : (DerivedPath) (DerivedPath::Opaque {

src/nix/app.cc

@@ -25,7 +25,8 @@ StringPairs resolveRewrites(
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
                 for (auto & [ outputName, outputPath ] : drvDep->outputs)
                     res.emplace(
-                        DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(),
+                        DownstreamPlaceholder::unknownCaOutput(
+                            drvDep->drvPath->outPath(), outputName).render(),
                         store.printStorePath(outputPath)
                     );
     return res;
@@ -65,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state)
                 [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath {
                     /* We want all outputs of the drv */
                     return DerivedPath::Built {
-                        .drvPath = d.drvPath,
+                        .drvPath = makeConstantStorePathRef(d.drvPath),
                         .outputs = OutputsSpec::All {},
                     };
                 },
@@ -106,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state)
         auto program = outPath + "/bin/" + mainProgram;
         return UnresolvedApp { App {
             .context = { DerivedPath::Built {
-                .drvPath = drvPath,
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .outputs = OutputsSpec::Names { outputName },
             } },
             .program = program,

src/nix/build.cc

@@ -9,18 +9,18 @@
 
 using namespace nix;
 
-nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store)
 {
     auto res = nlohmann::json::array();
     for (auto & t : paths) {
-        std::visit([&res, store](const auto & t) {
+        std::visit([&](const auto & t) {
             res.push_back(t.toJSON(store));
         }, t.raw());
     }
     return res;
 }
 
-nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store)
+static nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, const Store & store)
 {
     auto res = nlohmann::json::array();
     for (auto & b : buildables) {
@@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             printMissing(store, pathsToBuild, lvlError);
 
             if (json)
-                logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump());
+                logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump());
 
             return;
         }
@@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             installables,
             repair ? bmRepair : buildMode);
 
-        if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())

src/nix/bundle.cc

@@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand
 
         store->buildPaths({
             DerivedPath::Built {
-                .drvPath = drvPath,
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .outputs = OutputsSpec::All { },
             },
         });

src/nix/develop.cc

@@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     /* Build the derivation. */
     store->buildPaths(
         { DerivedPath::Built {
-            .drvPath = shellDrvPath,
+            .drvPath = makeConstantStorePathRef(shellDrvPath),
             .outputs = OutputsSpec::All { },
         }},
         bmNormal, evalStore);

src/nix/flake.cc

@@ -544,7 +544,7 @@ struct CmdFlakeCheck : FlakeCommand
                                             *attr2.value, attr2.pos);
                                         if (drvPath && attr_name == settings.thisSystem.get()) {
                                             drvPaths.push_back(DerivedPath::Built {
-                                                    .drvPath = *drvPath,
+                                                .drvPath = makeConstantStorePathRef(*drvPath),
                                                 .outputs = OutputsSpec::All { },
                                             });
                                         }

src/nix/log.cc

@@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand
 
         auto b = installable->toDerivedPath();
 
+        // For compat with CLI today, TODO revisit
+        auto oneUp = std::visit(overloaded {
+            [&](const DerivedPath::Opaque & bo) {
+                return make_ref<SingleDerivedPath>(bo);
+            },
+            [&](const DerivedPath::Built & bfd) {
+                return bfd.drvPath;
+            },
+        }, b.path.raw());
+        auto path = resolveDerivedPath(*store, *oneUp);
+
         RunPager pager;
         for (auto & sub : subs) {
             auto * logSubP = dynamic_cast<LogStore *>(&*sub);
@@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand
             }
             auto & logSub = *logSubP;
 
-            auto log = std::visit(overloaded {
+            auto log = logSub.getBuildLog(path);
-                [&](const DerivedPath::Opaque & bo) {
-                    return logSub.getBuildLog(bo.path);
-                },
-                [&](const DerivedPath::Built & bfd) {
-                    return logSub.getBuildLog(bfd.drvPath);
-                },
-            }, b.path.raw());
             if (!log) continue;
             stopProgressBar();
             printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri());

tests/build.sh

@@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e
   | grepQuiet "has 2 entries in its context. It should only have exactly one entry"
 
 nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status '
-  (.[0] | .path | match(".*multiple-outputs-e.drv"))
+  (.[0] | match(".*multiple-outputs-e.drv"))
 '
 
 # Test building from raw store path to drv not expression.

tests/dyn-drv/build-built-drv.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# In the corresponding nix file, we have two derivations: the first, named `hello`,
+# is a normal recursive derivation, while the second, named dependent, has the
+# new outputHashMode "text". Note that in "dependent", we don't refer to the
+# build output of `hello`, but only to the path of the drv file. For this reason,
+# we only need to:
+#
+# - instantiate `hello`
+# - build `producingDrv`
+# - check that the path of the output coincides with that of the original derivation
+
+out1=$(nix build -f ./text-hashed-output.nix hello --no-link)
+
+clearStore
+
+drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
+
+expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"

tests/dyn-drv/local.mk

@@ -1,6 +1,7 @@
 dyn-drv-tests := \
   $(d)/text-hashed-output.sh \
-  $(d)/recursive-mod-json.sh
+  $(d)/recursive-mod-json.sh \
+  $(d)/build-built-drv.sh
 
 install-tests-groups += dyn-drv
 

tests/test-libstoreconsumer/main.cc

@@ -23,7 +23,7 @@ int main (int argc, char **argv)
 
         std::vector<DerivedPath> paths {
             DerivedPath::Built {
-                .drvPath = store->parseStorePath(drvPath),
+                .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)),
                 .outputs = OutputsSpec::Names{"out"}
             }
         };