Merge pull request #6815 from obsidiansystems/better-wanted-outputs

`OutputSpec` for `DerivationGoal` and `DerivedPath`, today's `OutputSpec` -> `ExtendedOutputSpec`

src/libcmd/installables.cc

@@ -1,5 +1,6 @@
 #include "globals.hh"
 #include "installables.hh"
+#include "outputs-spec.hh"
 #include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
@@ -401,18 +402,6 @@ struct InstallableStorePath : Installable
     ref<Store> store;
     DerivedPath req;
 
-    InstallableStorePath(ref<Store> store, StorePath && storePath)
-        : store(store),
-          req(storePath.isDerivation()
-              ? (DerivedPath) DerivedPath::Built {
-                  .drvPath = std::move(storePath),
-                  .outputs = {},
-              }
-              : (DerivedPath) DerivedPath::Opaque {
-                  .path = std::move(storePath),
-              })
-    { }
-
     InstallableStorePath(ref<Store> store, DerivedPath && req)
         : store(store), req(std::move(req))
     { }
@@ -445,19 +434,19 @@ struct InstallableAttrPath : InstallableValue
     SourceExprCommand & cmd;
     RootValue v;
     std::string attrPath;
-    OutputsSpec outputsSpec;
+    ExtendedOutputsSpec extendedOutputsSpec;
 
     InstallableAttrPath(
         ref<EvalState> state,
         SourceExprCommand & cmd,
         Value * v,
         const std::string & attrPath,
-        OutputsSpec outputsSpec)
+        ExtendedOutputsSpec extendedOutputsSpec)
         : InstallableValue(state)
         , cmd(cmd)
         , v(allocRootValue(v))
         , attrPath(attrPath)
-        , outputsSpec(std::move(outputsSpec))
+        , extendedOutputsSpec(std::move(extendedOutputsSpec))
     { }
 
     std::string what() const override { return attrPath; }
@@ -480,30 +469,39 @@ struct InstallableAttrPath : InstallableValue
 
         // Backward compatibility hack: group results by drvPath. This
         // helps keep .all output together.
-        std::map<StorePath, DerivedPath::Built> byDrvPath;
+        std::map<StorePath, OutputsSpec> byDrvPath;
 
         for (auto & drvInfo : drvInfos) {
             auto drvPath = drvInfo.queryDrvPath();
             if (!drvPath)
                 throw Error("'%s' is not a derivation", what());
 
+            auto newOutputs = std::visit(overloaded {
+                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;
-
+                    for (auto & output : drvInfo.queryOutputs(false, true))
-            if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
-                outputsToInstall = *outputNames;
-            else
-                for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
                         outputsToInstall.insert(output.first);
+                    return OutputsSpec::Names { std::move(outputsToInstall) };
+                },
+                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
+                    return e;
+                },
+            }, extendedOutputsSpec.raw());
 
-            auto derivedPath = byDrvPath.emplace(*drvPath, DerivedPath::Built { .drvPath = *drvPath }).first;
+            auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
 
-            for (auto & output : outputsToInstall)
+            if (!didInsert)
-                derivedPath->second.outputs.insert(output);
+                iter->second = iter->second.union_(newOutputs);
         }
 
         DerivedPathsWithInfo res;
-        for (auto & [_, info] : byDrvPath)
+        for (auto & [drvPath, outputs] : byDrvPath)
-            res.push_back({ .path = { info } });
+            res.push_back({
+                .path = DerivedPath::Built {
+                    .drvPath = drvPath,
+                    .outputs = outputs,
+                },
+            });
 
         return res;
     }
@@ -580,7 +578,7 @@ InstallableFlake::InstallableFlake(
     ref<EvalState> state,
     FlakeRef && flakeRef,
     std::string_view fragment,
-    OutputsSpec outputsSpec,
+    ExtendedOutputsSpec extendedOutputsSpec,
     Strings attrPaths,
     Strings prefixes,
     const flake::LockFlags & lockFlags)
@@ -588,7 +586,7 @@ InstallableFlake::InstallableFlake(
       flakeRef(flakeRef),
       attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
       prefixes(fragment == "" ? Strings{} : prefixes),
-      outputsSpec(std::move(outputsSpec)),
+      extendedOutputsSpec(std::move(extendedOutputsSpec)),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -638,48 +636,47 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto drvPath = attr->forceDerivation();
 
-    std::set<std::string> outputsToInstall;
     std::optional<NixInt> priority;
 
+    if (attr->maybeGetAttr(state->sOutputSpecified)) {
+    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt();
+    }
+
+    return {{
+        .path = DerivedPath::Built {
+            .drvPath = std::move(drvPath),
+            .outputs = std::visit(overloaded {
+                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
+                    std::set<std::string> outputsToInstall;
                     if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
                         if (aOutputSpecified->getBool()) {
                             if (auto aOutputName = attr->maybeGetAttr("outputName"))
                                 outputsToInstall = { aOutputName->getString() };
                         }
-    }
+                    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
-
-    else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
                         if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
                             for (auto & s : aOutputsToInstall->getListOfStrings())
                                 outputsToInstall.insert(s);
-        if (auto aPriority = aMeta->maybeGetAttr("priority"))
-            priority = aPriority->getInt();
-    }
-
-    if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
-        outputsToInstall.clear();
-        if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
-            for (auto & s : aOutputs->getListOfStrings())
-                outputsToInstall.insert(s);
                     }
 
                     if (outputsToInstall.empty())
                         outputsToInstall.insert("out");
 
-    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+                    return OutputsSpec::Names { std::move(outputsToInstall) };
-        outputsToInstall = *outputNames;
+                },
-
+                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
-    return {{
+                    return e;
-        .path = DerivedPath::Built {
+                },
-            .drvPath = std::move(drvPath),
+            }, extendedOutputsSpec.raw()),
-            .outputs = std::move(outputsToInstall),
         },
         .info = {
             .priority = priority,
             .originalRef = flakeRef,
             .resolvedRef = getLockedFlake()->flake.lockedRef,
             .attrPath = attrPath,
-            .outputsSpec = outputsSpec,
+            .extendedOutputsSpec = extendedOutputsSpec,
         }
     }};
 }
@@ -796,12 +793,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         }
 
         for (auto & s : ss) {
-            auto [prefix, outputsSpec] = parseOutputsSpec(s);
+            auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s);
             result.push_back(
                 std::make_shared<InstallableAttrPath>(
                     state, *this, vFile,
-                    prefix == "." ? "" : prefix,
+                    prefix == "." ? "" : std::string { prefix },
-                    outputsSpec));
+                    extendedOutputsSpec));
         }
 
     } else {
@@ -809,24 +806,46 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
-            auto found = s.rfind('^');
+            auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s);
-            if (found != std::string::npos) {
+            // To avoid clang's pedantry
-                try {
+            auto prefix = std::move(prefix_);
-                    result.push_back(std::make_shared<InstallableStorePath>(
+            auto extendedOutputsSpec = std::move(extendedOutputsSpec_);
-                        store,
-                        DerivedPath::Built::parse(*store, s.substr(0, found), s.substr(found + 1))));
-                    continue;
-                } catch (BadStorePath &) {
-                } catch (...) {
-                    if (!ex)
-                        ex = std::current_exception();
-                }
-            }
 
-            found = s.find('/');
+            auto found = prefix.find('/');
             if (found != std::string::npos) {
                 try {
-                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+                    auto derivedPath = std::visit(overloaded {
+                        // If the user did not use ^, we treat the output more liberally.
+                        [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
+                            // First, we accept a symlink chain or an actual store path.
+                            auto storePath = store->followLinksToStorePath(prefix);
+                            // Second, we see if the store path ends in `.drv` to decide what sort
+                            // of derived path they want.
+                            //
+                            // This handling predates the `^` syntax. The `^*` in
+                            // `/nix/store/hash-foo.drv^*` unambiguously means "do the
+                            // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could
+                            // also unambiguously mean "do the DerivedPath::Opaque` case".
+                            //
+                            // Issue #7261 tracks reconsidering this `.drv` dispatching.
+                            return storePath.isDerivation()
+                                ? (DerivedPath) DerivedPath::Built {
+                                    .drvPath = std::move(storePath),
+                                    .outputs = OutputsSpec::All {},
+                                }
+                                : (DerivedPath) DerivedPath::Opaque {
+                                    .path = std::move(storePath),
+                                };
+                        },
+                        // If the user did use ^, we just do exactly what is written.
+                        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+                            return DerivedPath::Built {
+                                .drvPath = store->parseStorePath(prefix),
+                                .outputs = outputSpec,
+                            };
+                        },
+                    }, extendedOutputsSpec.raw());
+                    result.push_back(std::make_shared<InstallableStorePath>(store, std::move(derivedPath)));
                     continue;
                 } catch (BadStorePath &) {
                 } catch (...) {
@@ -836,13 +855,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
                         fragment,
-                        outputsSpec,
+                        extendedOutputsSpec,
                         getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));
@@ -917,32 +936,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
             for (auto & aux : backmap[path]) {
                 std::visit(overloaded {
                     [&](const DerivedPath::Built & bfd) {
-                        OutputPathMap outputs;
+                        auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
-                        auto drv = evalStore->readDerivation(bfd.drvPath);
-                        auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                        auto drvOutputs = drv.outputsAndOptPaths(*store);
-                        for (auto & output : bfd.outputs) {
-                            auto outputHash = get(outputHashes, output);
-                            if (!outputHash)
-                                throw Error(
-                                    "the derivation '%s' doesn't have an output named '%s'",
-                                    store->printStorePath(bfd.drvPath), output);
-                            if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
-                                DrvOutput outputId { *outputHash, output };
-                                auto realisation = store->queryRealisation(outputId);
-                                if (!realisation)
-                                    throw MissingRealisation(outputId);
-                                outputs.insert_or_assign(output, realisation->outPath);
-                            } else {
-                                // If ca-derivations isn't enabled, assume that
-                                // the output path is statically known.
-                                auto drvOutput = get(drvOutputs, output);
-                                assert(drvOutput);
-                                assert(drvOutput->second);
-                                outputs.insert_or_assign(
-                                    output, *drvOutput->second);
-                            }
-                        }
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built { bfd.drvPath, outputs },
                             .info = aux.info}});

src/libcmd/installables.hh

@@ -59,7 +59,7 @@ struct ExtraPathInfo
     std::optional<FlakeRef> resolvedRef;
     std::optional<std::string> attrPath;
     // FIXME: merge with DerivedPath's 'outputs' field?
-    std::optional<OutputsSpec> outputsSpec;
+    std::optional<ExtendedOutputsSpec> extendedOutputsSpec;
 };
 
 /* A derived path with any additional info that commands might
@@ -169,7 +169,7 @@ struct InstallableFlake : InstallableValue
     FlakeRef flakeRef;
     Strings attrPaths;
     Strings prefixes;
-    OutputsSpec outputsSpec;
+    ExtendedOutputsSpec extendedOutputsSpec;
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
@@ -178,7 +178,7 @@ struct InstallableFlake : InstallableValue
         ref<EvalState> state,
         FlakeRef && flakeRef,
         std::string_view fragment,
-        OutputsSpec outputsSpec,
+        ExtendedOutputsSpec extendedOutputsSpec,
         Strings attrPaths,
         Strings prefixes,
         const flake::LockFlags & lockFlags);

src/libcmd/repl.cc

@@ -641,7 +641,12 @@ bool NixRepl::processLine(std::string line)
         Path drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b" || command == ":bl") {
-            state->store->buildPaths({DerivedPath::Built{drvPath}});
+            state->store->buildPaths({
+                DerivedPath::Built {
+                    .drvPath = drvPath,
+                    .outputs = OutputsSpec::All { },
+                },
+            });
             auto drv = state->store->readDerivation(drvPath);
             logger->cout("\nThis derivation produced the following outputs:");
             for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) {

src/libexpr/flake/flakeref.cc

@@ -238,15 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
     return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
-std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
     bool isFlake)
 {
-    auto [prefix, outputsSpec] = parseOutputsSpec(url);
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url);
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake);
-    return {std::move(flakeRef), fragment, outputsSpec};
+    return {std::move(flakeRef), fragment, extendedOutputsSpec};
 }
 
 }

src/libexpr/flake/flakeref.hh

@@ -80,7 +80,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
-std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,

src/libexpr/primops.cc

@@ -53,7 +53,7 @@ StringMap EvalState::realiseContext(const PathSet & context)
             [&](const NixStringContextElem::Built & b) {
                 drvs.push_back(DerivedPath::Built {
                     .drvPath = b.drvPath,
-                    .outputs = std::set { b.output },
+                    .outputs = OutputsSpec::Names { b.output },
                 });
                 ensureValid(b.drvPath);
             },
@@ -84,16 +84,12 @@ StringMap EvalState::realiseContext(const PathSet & context)
     store->buildPaths(buildReqs);
 
     /* Get all the output paths corresponding to the placeholders we had */
-    for (auto & [drvPath, outputs] : drvs) {
+    for (auto & drv : drvs) {
-        const auto outputPaths = store->queryDerivationOutputMap(drvPath);
+        auto outputs = resolveDerivedPath(*store, drv);
-        for (auto & outputName : outputs) {
+        for (auto & [outputName, outputPath] : outputs) {
-            auto outputPath = get(outputPaths, outputName);
-            if (!outputPath)
-                debugThrowLastTrace(Error("derivation '%s' does not have an output named '%s'",
-                    store->printStorePath(drvPath), outputName));
             res.insert_or_assign(
-                downstreamPlaceholder(*store, drvPath, outputName),
+                downstreamPlaceholder(*store, drv.drvPath, outputName),
-                store->printStorePath(*outputPath)
+                store->printStorePath(outputPath)
             );
         }
     }

src/libstore/build/derivation-goal.cc

@@ -63,7 +63,7 @@
 namespace nix {
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
+    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
     , useDerivation(true)
     , drvPath(drvPath)
@@ -82,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
 
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
+    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
     , useDerivation(false)
     , drvPath(drvPath)
@@ -142,18 +142,12 @@ void DerivationGoal::work()
     (this->*state)();
 }
 
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
+void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 {
-    /* If we already want all outputs, there is nothing to do. */
+    auto newWanted = wantedOutputs.union_(outputs);
-    if (wantedOutputs.empty()) return;
+    if (!newWanted.isSubsetOf(wantedOutputs))
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
         needRestart = true;
+    wantedOutputs = newWanted;
 }
 
 
@@ -390,7 +384,7 @@ void DerivationGoal::repairClosure()
     auto outputs = queryDerivationOutputMap();
     StorePathSet outputClosure;
     for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
+        if (!wantedOutputs.contains(i.first)) continue;
         worker.store.computeFSClosure(i.second, outputClosure);
     }
 
@@ -422,7 +416,7 @@ void DerivationGoal::repairClosure()
         if (drvPath2 == outputsToDrv.end())
             addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
+            addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
     }
 
     if (waitees.empty()) {
@@ -991,10 +985,15 @@ void DerivationGoal::resolvedFinished()
 
         StorePathSet outputPaths;
 
-        // `wantedOutputs` might be empty, which means “all the outputs”
+        // `wantedOutputs` might merely indicate “all the outputs”
-        auto realWantedOutputs = wantedOutputs;
+        auto realWantedOutputs = std::visit(overloaded {
-        if (realWantedOutputs.empty())
+            [&](const OutputsSpec::All &) {
-            realWantedOutputs = resolvedDrv.outputNames();
+                return resolvedDrv.outputNames();
+            },
+            [&](const OutputsSpec::Names & names) {
+                return static_cast<std::set<std::string>>(names);
+            },
+        }, wantedOutputs.raw());
 
         for (auto & wantedOutput : realWantedOutputs) {
             auto initialOutput = get(initialOutputs, wantedOutput);
@@ -1322,7 +1321,14 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
     if (!drv->type().isPure()) return { false, {} };
 
     bool checkHash = buildMode == bmRepair;
-    auto wantedOutputsLeft = wantedOutputs;
+    auto wantedOutputsLeft = std::visit(overloaded {
+        [&](const OutputsSpec::All &) {
+            return StringSet {};
+        },
+        [&](const OutputsSpec::Names & names) {
+            return static_cast<StringSet>(names);
+        },
+    }, wantedOutputs.raw());
     DrvOutputs validOutputs;
 
     for (auto & i : queryPartialDerivationOutputMap()) {
@@ -1331,7 +1337,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
             // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
             continue;
         auto & info = *initialOutput;
-        info.wanted = wantOutput(i.first, wantedOutputs);
+        info.wanted = wantedOutputs.contains(i.first);
         if (info.wanted)
             wantedOutputsLeft.erase(i.first);
         if (i.second) {
@@ -1369,7 +1375,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
             validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
     }
 
-    // If we requested all the outputs via the empty set, we are always fine.
+    // If we requested all the outputs, we are always fine.
     // If we requested specific elements, the loop above removes all the valid
     // ones, so any that are left must be invalid.
     if (!wantedOutputsLeft.empty())

src/libstore/build/derivation-goal.hh

@@ -2,6 +2,7 @@
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
+#include "outputs-spec.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "goal.hh"
@@ -55,7 +56,7 @@ struct DerivationGoal : public Goal
 
     /* The specific outputs that we need to build.  Empty means all of
        them. */
-    StringSet wantedOutputs;
+    OutputsSpec wantedOutputs;
 
     /* Mapping from input derivations + output names to actual store
        paths. This is filled in by waiteeDone() as each dependency
@@ -128,10 +129,10 @@ struct DerivationGoal : public Goal
     std::string machineName;
 
     DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
+        const OutputsSpec & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
     DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
+        const OutputsSpec & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
     virtual ~DerivationGoal();
 
@@ -142,7 +143,7 @@ struct DerivationGoal : public Goal
     void work() override;
 
     /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
+    void addWantedOutputs(const OutputsSpec & outputs);
 
     /* The states. */
     void getDerivation();

src/libstore/build/entry-points.cc

@@ -80,7 +80,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     BuildMode buildMode)
 {
     Worker worker(*this, *this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
+    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
 
     try {
         worker.run(Goals{goal});
@@ -89,7 +89,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
         return BuildResult {
             .status = BuildResult::MiscFailure,
             .errorMsg = e.msg(),
-            .path = DerivedPath::Built { .drvPath = drvPath },
+            .path = DerivedPath::Built {
+                .drvPath = drvPath,
+                .outputs = OutputsSpec::All { },
+            },
         };
     };
 }
@@ -130,7 +133,8 @@ void LocalStore::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
+            // FIXME: Should just build the specific output we need.
+            goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
             worker.run(goals);
         } else
             throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));

src/libstore/build/local-derivation-goal.cc

@@ -2735,7 +2735,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
-        if (wantOutput(outputName, wantedOutputs))
+        if (wantedOutputs.contains(outputName))
             builtOutputs.emplace(thisRealisation.id, thisRealisation);
     }
 

src/libstore/build/worker.cc

@@ -42,7 +42,7 @@ Worker::~Worker()
 
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
-    const StringSet & wantedOutputs,
+    const OutputsSpec & wantedOutputs,
     std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
     std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
@@ -59,7 +59,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
 
 
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
+    const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
         return !dynamic_cast<LocalStore *>(&store)
@@ -70,7 +70,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 
 
 std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
+    const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
         return !dynamic_cast<LocalStore *>(&store)

src/libstore/build/worker.hh

@@ -140,15 +140,15 @@ public:
     /* derivation goal */
 private:
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
+        const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
 public:
     std::shared_ptr<DerivationGoal> makeDerivationGoal(
         const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
         const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
 
     /* substitution goal */
     std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);

src/libstore/derivations.cc

@@ -688,12 +688,6 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
 }
 
 
-bool wantOutput(const std::string & output, const std::set<std::string> & wanted)
-{
-    return wanted.empty() || wanted.find(output) != wanted.end();
-}
-
-
 static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
     const auto pathS = readString(in);

src/libstore/derivations.hh

@@ -294,8 +294,6 @@ typedef std::map<StorePath, DrvHash> DrvHashes;
 // FIXME: global, though at least thread-safe.
 extern Sync<DrvHashes> drvHashes;
 
-bool wantOutput(const std::string & output, const std::set<std::string> & wanted);
-
 struct Source;
 struct Sink;
 

src/libstore/derived-path.cc

@@ -19,11 +19,11 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
     res["drvPath"] = store->printStorePath(drvPath);
     // Fallback for the input-addressed derivation case: We expect to always be
     // able to print the output paths, so let’s do it
-    const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
+    const auto outputMap = store->queryPartialDerivationOutputMap(drvPath);
-    for (const auto & output : outputs) {
+    for (const auto & [output, outputPathOpt] : outputMap) {
-        auto knownOutput = get(knownOutputs, output);
+        if (!outputs.contains(output)) continue;
-        if (knownOutput && *knownOutput)
+        if (outputPathOpt)
-            res["outputs"][output] = store->printStorePath(**knownOutput);
+            res["outputs"][output] = store->printStorePath(*outputPathOpt);
         else
             res["outputs"][output] = nullptr;
     }
@@ -63,7 +63,7 @@ std::string DerivedPath::Built::to_string(const Store & store) const
 {
     return store.printStorePath(drvPath)
         + "!"
-        + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
+        + outputs.to_string();
 }
 
 std::string DerivedPath::to_string(const Store & store) const
@@ -81,15 +81,10 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
 
 DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
 {
-    auto drvPath = store.parseStorePath(drvS);
+    return {
-    std::set<std::string> outputs;
+        .drvPath = store.parseStorePath(drvS),
-    if (outputsS != "*") {
+        .outputs = OutputsSpec::parse(outputsS),
-        outputs = tokenizeString<std::set<std::string>>(outputsS, ",");
+    };
-        if (outputs.empty())
-            throw Error(
-                 "Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS);
-	}
-    return {drvPath, outputs};
 }
 
 DerivedPath DerivedPath::parse(const Store & store, std::string_view s)

src/libstore/derived-path.hh

@@ -3,6 +3,7 @@
 #include "util.hh"
 #include "path.hh"
 #include "realisation.hh"
+#include "outputs-spec.hh"
 
 #include <optional>
 
@@ -44,7 +45,7 @@ struct DerivedPathOpaque {
  */
 struct DerivedPathBuilt {
     StorePath drvPath;
-    std::set<std::string> outputs;
+    OutputsSpec outputs;
 
     std::string to_string(const Store & store) const;
     static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);

src/libstore/legacy-ssh-store.cc

@@ -279,7 +279,12 @@ public:
 
         conn->to.flush();
 
-        BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } };
+        BuildResult status {
+            .path = DerivedPath::Built {
+                .drvPath = drvPath,
+                .outputs = OutputsSpec::All { },
+            },
+        };
         status.status = (BuildResult::Status) readInt(conn->from);
         conn->from >> status.errorMsg;
 

src/libstore/misc.cc

@@ -185,7 +185,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
                     knownOutputPaths = false;
                     break;
                 }
-                if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
+                if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt))
                     invalid.insert(*pathOpt);
             }
             if (knownOutputPaths && invalid.empty()) return;
@@ -301,4 +301,47 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
     return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
 }
 
+OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
+{
+    auto & evalStore = evalStore_ ? *evalStore_ : store;
+
+    OutputPathMap outputs;
+    auto drv = evalStore.readDerivation(bfd.drvPath);
+    auto outputHashes = staticOutputHashes(store, drv);
+    auto drvOutputs = drv.outputsAndOptPaths(store);
+    auto outputNames = std::visit(overloaded {
+        [&](const OutputsSpec::All &) {
+            StringSet names;
+            for (auto & [outputName, _] : drv.outputs)
+                names.insert(outputName);
+            return names;
+        },
+        [&](const OutputsSpec::Names & names) {
+            return static_cast<std::set<std::string>>(names);
+        },
+    }, bfd.outputs);
+    for (auto & output : outputNames) {
+        auto outputHash = get(outputHashes, output);
+        if (!outputHash)
+            throw Error(
+                "the derivation '%s' doesn't have an output named '%s'",
+                store.printStorePath(bfd.drvPath), output);
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+            DrvOutput outputId { *outputHash, output };
+            auto realisation = store.queryRealisation(outputId);
+            if (!realisation)
+                throw MissingRealisation(outputId);
+            outputs.insert_or_assign(output, realisation->outPath);
+        } else {
+            // If ca-derivations isn't enabled, assume that
+            // the output path is statically known.
+            auto drvOutput = get(drvOutputs, output);
+            assert(drvOutput);
+            assert(drvOutput->second);
+            outputs.insert_or_assign(output, *drvOutput->second);
+        }
+    }
+    return outputs;
+}
+
 }

src/libstore/outputs-spec.cc

@@ -1,3 +1,4 @@
+#include "util.hh"
 #include "outputs-spec.hh"
 #include "nlohmann/json.hpp"
 
@@ -5,57 +6,184 @@
 
 namespace nix {
 
-std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
+bool OutputsSpec::contains(const std::string & outputName) const
 {
-    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
+    return std::visit(overloaded {
-
+        [&](const OutputsSpec::All &) {
-    std::smatch match;
+            return true;
-    if (!std::regex_match(s, match, regex))
+        },
-        return {s, DefaultOutputs()};
+        [&](const OutputsSpec::Names & outputNames) {
-
+            return outputNames.count(outputName) > 0;
-    if (match[3].matched)
+        },
-        return {match[1], AllOutputs()};
+    }, raw());
-
-    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
 }
 
-std::string printOutputsSpec(const OutputsSpec & outputsSpec)
+
+std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s)
 {
-    if (std::get_if<DefaultOutputs>(&outputsSpec))
+    static std::regex regex(R"((\*)|([a-z]+(,[a-z]+)*))");
-        return "";
 
-    if (std::get_if<AllOutputs>(&outputsSpec))
+    std::smatch match;
-        return "^*";
+    std::string s2 { s }; // until some improves std::regex
+    if (!std::regex_match(s2, match, regex))
+        return std::nullopt;
 
-    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+    if (match[1].matched)
-        return "^" + concatStringsSep(",", *outputNames);
+        return { OutputsSpec::All {} };
+
+    if (match[2].matched)
+        return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") };
 
     assert(false);
 }
 
-void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
+
+OutputsSpec OutputsSpec::parse(std::string_view s)
 {
-    if (std::get_if<DefaultOutputs>(&outputsSpec))
+    std::optional spec = parseOpt(s);
-        json = nullptr;
+    if (!spec)
-
+        throw Error("Invalid outputs specifier: '%s'", s);
-    else if (std::get_if<AllOutputs>(&outputsSpec))
+    return *spec;
-        json = std::vector<std::string>({"*"});
-
-    else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
-        json = *outputNames;
 }
 
-void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
+
+std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsSpec::parseOpt(std::string_view s)
 {
-    if (json.is_null())
+    auto found = s.rfind('^');
-        outputsSpec = DefaultOutputs();
+
-    else {
+    if (found == std::string::npos)
-        auto names = json.get<OutputNames>();
+        return std::pair { s, ExtendedOutputsSpec::Default {} };
-        if (names == OutputNames({"*"}))
+
-            outputsSpec = AllOutputs();
+    auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1));
+    if (!specOpt)
+        return std::nullopt;
+    return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } };
+}
+
+
+std::pair<std::string_view, ExtendedOutputsSpec> ExtendedOutputsSpec::parse(std::string_view s)
+{
+    std::optional spec = parseOpt(s);
+    if (!spec)
+        throw Error("Invalid extended outputs specifier: '%s'", s);
+    return *spec;
+}
+
+
+std::string OutputsSpec::to_string() const
+{
+    return std::visit(overloaded {
+        [&](const OutputsSpec::All &) -> std::string {
+            return "*";
+        },
+        [&](const OutputsSpec::Names & outputNames) -> std::string {
+            return concatStringsSep(",", outputNames);
+        },
+    }, raw());
+}
+
+
+std::string ExtendedOutputsSpec::to_string() const
+{
+    return std::visit(overloaded {
+        [&](const ExtendedOutputsSpec::Default &) -> std::string {
+            return "";
+        },
+        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string {
+            return "^" + outputSpec.to_string();
+        },
+    }, raw());
+}
+
+
+OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const
+{
+    return std::visit(overloaded {
+        [&](const OutputsSpec::All &) -> OutputsSpec {
+            return OutputsSpec::All { };
+        },
+        [&](const OutputsSpec::Names & theseNames) -> OutputsSpec {
+            return std::visit(overloaded {
+                [&](const OutputsSpec::All &) -> OutputsSpec {
+                    return OutputsSpec::All {};
+                },
+                [&](const OutputsSpec::Names & thoseNames) -> OutputsSpec {
+                    OutputsSpec::Names ret = theseNames;
+                    ret.insert(thoseNames.begin(), thoseNames.end());
+                    return ret;
+                },
+            }, that.raw());
+        },
+    }, raw());
+}
+
+
+bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const
+{
+    return std::visit(overloaded {
+        [&](const OutputsSpec::All &) {
+            return true;
+        },
+        [&](const OutputsSpec::Names & thoseNames) {
+            return std::visit(overloaded {
+                [&](const OutputsSpec::All &) {
+                    return false;
+                },
+                [&](const OutputsSpec::Names & theseNames) {
+                    bool ret = true;
+                    for (auto & o : theseNames)
+                        if (thoseNames.count(o) == 0)
+                            ret = false;
+                    return ret;
+                },
+            }, raw());
+        },
+    }, that.raw());
+}
+
+}
+
+namespace nlohmann {
+
+using namespace nix;
+
+OutputsSpec adl_serializer<OutputsSpec>::from_json(const json & json) {
+    auto names = json.get<StringSet>();
+    if (names == StringSet({"*"}))
+        return OutputsSpec::All {};
     else
-            outputsSpec = names;
+        return OutputsSpec::Names { std::move(names) };
+}
+
+void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) {
+    std::visit(overloaded {
+        [&](const OutputsSpec::All &) {
+            json = std::vector<std::string>({"*"});
+        },
+        [&](const OutputsSpec::Names & names) {
+            json = names;
+        },
+    }, t);
+}
+
+
+ExtendedOutputsSpec adl_serializer<ExtendedOutputsSpec>::from_json(const json & json) {
+    if (json.is_null())
+        return ExtendedOutputsSpec::Default {};
+    else {
+        return ExtendedOutputsSpec::Explicit { json.get<OutputsSpec>() };
     }
 }
 
+void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSpec t) {
+    std::visit(overloaded {
+        [&](const ExtendedOutputsSpec::Default &) {
+            json = nullptr;
+        },
+        [&](const ExtendedOutputsSpec::Explicit & e) {
+            adl_serializer<OutputsSpec>::to_json(json, e);
+        },
+    }, t);
+}
+
 }

src/libstore/outputs-spec.hh

@@ -1,32 +1,95 @@
 #pragma once
 
+#include <cassert>
+#include <optional>
+#include <set>
 #include <variant>
 
-#include "util.hh"
+#include "json-impls.hh"
-
-#include "nlohmann/json_fwd.hpp"
 
 namespace nix {
 
-typedef std::set<std::string> OutputNames;
+struct OutputNames : std::set<std::string> {
+    using std::set<std::string>::set;
 
-struct AllOutputs {
+    /* These need to be "inherited manually" */
-    bool operator < (const AllOutputs & _) const { return false; }
+
+    OutputNames(const std::set<std::string> & s)
+        : std::set<std::string>(s)
+    { assert(!empty()); }
+
+    OutputNames(std::set<std::string> && s)
+        : std::set<std::string>(s)
+    { assert(!empty()); }
+
+    /* This set should always be non-empty, so we delete this
+       constructor in order make creating empty ones by mistake harder.
+       */
+    OutputNames() = delete;
 };
 
-struct DefaultOutputs {
+struct AllOutputs : std::monostate { };
-    bool operator < (const DefaultOutputs & _) const { return false; }
+
+typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw;
+
+struct OutputsSpec : _OutputsSpecRaw {
+    using Raw = _OutputsSpecRaw;
+    using Raw::Raw;
+
+    /* Force choosing a variant */
+    OutputsSpec() = delete;
+
+    using Names = OutputNames;
+    using All = AllOutputs;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    inline Raw & raw() {
+        return static_cast<Raw &>(*this);
+    }
+
+    bool contains(const std::string & output) const;
+
+    /* Create a new OutputsSpec which is the union of this and that. */
+    OutputsSpec union_(const OutputsSpec & that) const;
+
+    /* Whether this OutputsSpec is a subset of that. */
+    bool isSubsetOf(const OutputsSpec & outputs) const;
+
+    /* Parse a string of the form 'output1,...outputN' or
+       '*', returning the outputs spec. */
+    static OutputsSpec parse(std::string_view s);
+    static std::optional<OutputsSpec> parseOpt(std::string_view s);
+
+    std::string to_string() const;
 };
 
-typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
+struct DefaultOutputs : std::monostate { };
+
+typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw;
+
+struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw {
+    using Raw = _ExtendedOutputsSpecRaw;
+    using Raw::Raw;
+
+    using Default = DefaultOutputs;
+    using Explicit = OutputsSpec;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
 
     /* Parse a string of the form 'prefix^output1,...outputN' or
-   'prefix^*', returning the prefix and the outputs spec. */
+       'prefix^*', returning the prefix and the extended outputs spec. */
-std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
+    static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s);
+    static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s);
 
-std::string printOutputsSpec(const OutputsSpec & outputsSpec);
+    std::string to_string() const;
-
+};
-void to_json(nlohmann::json &, const OutputsSpec &);
-void from_json(const nlohmann::json &, OutputsSpec &);
 
 }
+
+JSON_IMPL(OutputsSpec)
+JSON_IMPL(ExtendedOutputsSpec)

src/libstore/path-with-outputs.cc

@@ -15,11 +15,15 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
 
 DerivedPath StorePathWithOutputs::toDerivedPath() const
 {
-    if (!outputs.empty() || path.isDerivation())
+    if (!outputs.empty()) {
-        return DerivedPath::Built { path, outputs };
+        return DerivedPath::Built { path, OutputsSpec::Names { outputs } };
-    else
+    } else if (path.isDerivation()) {
+        assert(outputs.empty());
+        return DerivedPath::Built { path, OutputsSpec::All { } };
+    } else {
         return DerivedPath::Opaque { path };
     }
+}
 
 
 std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
@@ -41,7 +45,18 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv
             return StorePathWithOutputs { bo.path };
         },
         [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
-            return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
+            return StorePathWithOutputs {
+                .path = bfd.drvPath,
+                // Use legacy encoding of wildcard as empty set
+                .outputs = std::visit(overloaded {
+                    [&](const OutputsSpec::All &) -> StringSet {
+                        return {};
+                    },
+                    [&](const OutputsSpec::Names & outputs) {
+                        return static_cast<StringSet>(outputs);
+                    },
+                }, bfd.outputs.raw()),
+            };
         },
     }, p.raw());
 }

src/libstore/path-with-outputs.hh

@@ -2,7 +2,6 @@
 
 #include "path.hh"
 #include "derived-path.hh"
-#include "nlohmann/json_fwd.hpp"
 
 namespace nix {
 

src/libstore/path.hh

@@ -64,7 +64,6 @@ public:
 
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
-typedef std::map<std::string, StorePath> OutputPathMap;
 
 typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 

src/libstore/remote-store.cc

@@ -867,8 +867,8 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
                         OutputPathMap outputs;
                         auto drv = evalStore->readDerivation(bfd.drvPath);
                         const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                        const auto drvOutputs = drv.outputsAndOptPaths(*this);
+                        auto built = resolveDerivedPath(*this, bfd, &*evalStore);
-                        for (auto & output : bfd.outputs) {
+                        for (auto & [output, outputPath] : built) {
                             auto outputHash = get(outputHashes, output);
                             if (!outputHash)
                                 throw Error(
@@ -882,16 +882,11 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
                                     throw MissingRealisation(outputId);
                                 res.builtOutputs.emplace(realisation->id, *realisation);
                             } else {
-                                // If ca-derivations isn't enabled, assume that
-                                // the output path is statically known.
-                                const auto drvOutput = get(drvOutputs, output);
-                                assert(drvOutput);
-                                assert(drvOutput->second);
                                 res.builtOutputs.emplace(
                                     outputId,
                                     Realisation {
                                         .id = outputId,
-                                        .outPath = *drvOutput->second,
+                                        .outPath = outputPath,
                                     });
                             }
                         }
@@ -915,7 +910,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     writeDerivation(conn->to, *this, drv);
     conn->to << buildMode;
     conn.processStderr();
-    BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } };
+    BuildResult res {
+        .path = DerivedPath::Built {
+            .drvPath = drvPath,
+            .outputs = OutputsSpec::All { },
+        },
+    };
     res.status = (BuildResult::Status) readInt(conn->from);
     conn->from >> res.errorMsg;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {

src/libstore/store-api.hh

@@ -71,6 +71,9 @@ class NarInfoDiskCache;
 class Store;
 
 
+typedef std::map<std::string, StorePath> OutputPathMap;
+
+
 enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
 enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@@ -120,6 +123,8 @@ public:
 
     typedef std::map<std::string, std::string> Params;
 
+
+
 protected:
 
     struct PathInfoCacheValue {
@@ -726,6 +731,11 @@ void copyClosure(
 void removeTempRoots();
 
 
+/* Resolve the derived path completely, failing if any derivation output
+   is unknown. */
+OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);
+
+
 /* Return a Store object to access the Nix store denoted by
    ‘uri’ (slight misnomer...). Supported values are:
 

src/libstore/tests/outputs-spec.cc

@@ -1,46 +1,187 @@
 #include "outputs-spec.hh"
 
+#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
 namespace nix {
 
-TEST(parseOutputsSpec, basic)
+#ifndef NDEBUG
-{
+TEST(OutputsSpec, no_empty_names) {
-    {
+    ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, "");
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo");
+}
-        ASSERT_EQ(prefix, "foo");
+#endif
-        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+
+#define TEST_DONT_PARSE(NAME, STR)           \
+    TEST(OutputsSpec, bad_ ## NAME) {        \
+        std::optional OutputsSpecOpt =       \
+            OutputsSpec::parseOpt(STR);      \
+        ASSERT_FALSE(OutputsSpecOpt);        \
     }
 
-    {
+TEST_DONT_PARSE(empty, "")
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
+TEST_DONT_PARSE(garbage, "&*()")
-        ASSERT_EQ(prefix, "foo");
+TEST_DONT_PARSE(double_star, "**")
-        ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
+TEST_DONT_PARSE(star_first, "*,foo")
+TEST_DONT_PARSE(star_second, "foo,*")
+
+#undef TEST_DONT_PARSE
+
+TEST(OutputsSpec, all) {
+    std::string_view str = "*";
+    OutputsSpec expected = OutputsSpec::All { };
+    ASSERT_EQ(OutputsSpec::parse(str), expected);
+    ASSERT_EQ(expected.to_string(), str);
 }
 
-    {
+TEST(OutputsSpec, names_out) {
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
+    std::string_view str = "out";
-        ASSERT_EQ(prefix, "foo");
+    OutputsSpec expected = OutputsSpec::Names { "out" };
-        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
+    ASSERT_EQ(OutputsSpec::parse(str), expected);
+    ASSERT_EQ(expected.to_string(), str);
 }
 
-    {
+TEST(OutputsSpec, names_out_bin) {
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
+    OutputsSpec expected = OutputsSpec::Names { "out", "bin" };
-        ASSERT_EQ(prefix, "foo");
+    ASSERT_EQ(OutputsSpec::parse("out,bin"), expected);
-        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    // N.B. This normalization is OK.
+    ASSERT_EQ(expected.to_string(), "bin,out");
 }
 
-    {
+#define TEST_SUBSET(X, THIS, THAT) \
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
+    X((OutputsSpec { THIS }).isSubsetOf(THAT));
+
+TEST(OutputsSpec, subsets_all_all) {
+    TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { });
+}
+
+TEST(OutputsSpec, subsets_names_all) {
+    TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { });
+}
+
+TEST(OutputsSpec, subsets_names_names_eq) {
+    TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" });
+}
+
+TEST(OutputsSpec, subsets_names_names_noneq) {
+    TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" }));
+}
+
+TEST(OutputsSpec, not_subsets_all_names) {
+    TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" });
+}
+
+TEST(OutputsSpec, not_subsets_names_names) {
+    TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" }));
+}
+
+#undef TEST_SUBSET
+
+#define TEST_UNION(RES, THIS, THAT) \
+    ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT));
+
+TEST(OutputsSpec, union_all_all) {
+    TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { });
+}
+
+TEST(OutputsSpec, union_all_names) {
+    TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" });
+}
+
+TEST(OutputsSpec, union_names_all) {
+    TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { });
+}
+
+TEST(OutputsSpec, union_names_names) {
+    TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" });
+}
+
+#undef TEST_UNION
+
+#define TEST_DONT_PARSE(NAME, STR)                   \
+    TEST(ExtendedOutputsSpec, bad_ ## NAME) {        \
+        std::optional extendedOutputsSpecOpt =       \
+            ExtendedOutputsSpec::parseOpt(STR);      \
+        ASSERT_FALSE(extendedOutputsSpecOpt);        \
+    }
+
+TEST_DONT_PARSE(carot_empty, "^")
+TEST_DONT_PARSE(prefix_carot_empty, "foo^")
+TEST_DONT_PARSE(garbage, "^&*()")
+TEST_DONT_PARSE(double_star, "^**")
+TEST_DONT_PARSE(star_first, "^*,foo")
+TEST_DONT_PARSE(star_second, "^foo,*")
+
+#undef TEST_DONT_PARSE
+
+TEST(ExtendedOutputsSpec, defeault) {
+    std::string_view str = "foo";
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
+    ASSERT_EQ(prefix, "foo");
+    ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { };
+    ASSERT_EQ(extendedOutputsSpec, expected);
+    ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
+}
+
+TEST(ExtendedOutputsSpec, all) {
+    std::string_view str = "foo^*";
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
+    ASSERT_EQ(prefix, "foo");
+    ExtendedOutputsSpec expected = OutputsSpec::All { };
+    ASSERT_EQ(extendedOutputsSpec, expected);
+    ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
+}
+
+TEST(ExtendedOutputsSpec, out) {
+    std::string_view str = "foo^out";
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
+    ASSERT_EQ(prefix, "foo");
+    ExtendedOutputsSpec expected = OutputsSpec::Names { "out" };
+    ASSERT_EQ(extendedOutputsSpec, expected);
+    ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
+}
+
+TEST(ExtendedOutputsSpec, out_bin) {
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin");
+    ASSERT_EQ(prefix, "foo");
+    ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
+    ASSERT_EQ(extendedOutputsSpec, expected);
+    ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out");
+}
+
+TEST(ExtendedOutputsSpec, many_carrot) {
+    auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin");
     ASSERT_EQ(prefix, "foo^bar");
-        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
+    ASSERT_EQ(extendedOutputsSpec, expected);
+    ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out");
 }
 
-    {
+
-        auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
+#define TEST_JSON(TYPE, NAME, STR, VAL)                    \
-        ASSERT_EQ(prefix, "foo^&*()");
+                                                           \
-        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    TEST(TYPE, NAME ## _to_json) {                         \
-    }
+        using nlohmann::literals::operator "" _json;       \
+        ASSERT_EQ(                                         \
+            STR ## _json,                                  \
+            ((nlohmann::json) TYPE { VAL }));              \
+    }                                                      \
+                                                           \
+    TEST(TYPE, NAME ## _from_json) {                       \
+        using nlohmann::literals::operator "" _json;       \
+        ASSERT_EQ(                                         \
+            TYPE { VAL },                                  \
+            (STR ## _json).get<TYPE>());                   \
     }
 
+TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { })
+TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" })
+TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" }))
+
+TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { })
+TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } })
+TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } })
+TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } }))
+
+#undef TEST_JSON
+
 }

src/libutil/json-impls.hh

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "nlohmann/json_fwd.hpp"
+
+// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
+#define JSON_IMPL(TYPE)                                                \
+    namespace nlohmann {                                               \
+        using namespace nix;                                           \
+        template <>                                                    \
+        struct adl_serializer<TYPE> {                                  \
+            static TYPE from_json(const json & json);                  \
+            static void to_json(json & json, TYPE t);                  \
+        };                                                             \
+    }

src/nix-build/nix-build.cc

@@ -397,7 +397,7 @@ static void main_nix_build(int argc, char * * argv)
                 auto bashDrv = drv->requireDrvPath();
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = bashDrv,
-                    .outputs = {"out"},
+                    .outputs = OutputsSpec::Names {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
                 shellDrv = bashDrv;
@@ -421,7 +421,7 @@ static void main_nix_build(int argc, char * * argv)
             {
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = inputDrv,
-                    .outputs = inputOutputs
+                    .outputs = OutputsSpec::Names { inputOutputs },
                 });
                 pathsToCopy.insert(inputDrv);
             }
@@ -591,7 +591,7 @@ static void main_nix_build(int argc, char * * argv)
             if (outputName == "")
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
-            pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}});
+            pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
             pathsToBuildOrdered.push_back({drvPath, {outputName}});
             drvsToCopy.insert(drvPath);
 

src/nix-env/nix-env.cc

@@ -478,9 +478,14 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     std::vector<DerivedPath> targets;
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath())
-            targets.push_back(DerivedPath::Built{*drvPath});
+            targets.push_back(DerivedPath::Built{
+                .drvPath = *drvPath,
+                .outputs = OutputsSpec::All { },
+            });
         else
-            targets.push_back(DerivedPath::Opaque{i.queryOutPath()});
+            targets.push_back(DerivedPath::Opaque{
+                .path = i.queryOutPath(),
+            });
 
     printMissing(state.store, targets);
 }
@@ -751,8 +756,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     auto drvPath = drv.queryDrvPath();
     std::vector<DerivedPath> paths {
         drvPath
-        ? (DerivedPath) (DerivedPath::Built { *drvPath })
+        ? (DerivedPath) (DerivedPath::Built {
-        : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }),
+            .drvPath = *drvPath,
+            .outputs = OutputsSpec::All { },
+        })
+        : (DerivedPath) (DerivedPath::Opaque {
+            .path = drv.queryOutPath(),
+        }),
     };
     printMissing(globals.state->store, paths);
     if (globals.dryRun) return;

src/nix/app.cc

@@ -86,13 +86,13 @@ UnresolvedApp Installable::toApp(EvalState & state)
                     /* We want all outputs of the drv */
                     return DerivedPath::Built {
                         .drvPath = d.drvPath,
-                        .outputs = {},
+                        .outputs = OutputsSpec::All {},
                     };
                 },
                 [&](const NixStringContextElem::Built & b) -> DerivedPath {
                     return DerivedPath::Built {
                         .drvPath = b.drvPath,
-                        .outputs = { b.output },
+                        .outputs = OutputsSpec::Names { b.output },
                     };
                 },
                 [&](const NixStringContextElem::Opaque & o) -> DerivedPath {
@@ -127,7 +127,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
         return UnresolvedApp { App {
             .context = { DerivedPath::Built {
                 .drvPath = drvPath,
-                .outputs = {outputName},
+                .outputs = OutputsSpec::Names { outputName },
             } },
             .program = program,
         }};

src/nix/bundle.cc

@@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
 
         auto val = installable->toValue(*evalState).first;
 
-        auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
+        auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
         InstallableFlake bundler{this,
-            evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
+            evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec,
             {"bundlers." + settings.thisSystem.get() + ".default",
              "defaultBundler." + settings.thisSystem.get()
             },
@@ -105,7 +105,12 @@ struct CmdBundle : InstallableCommand
 
         auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, "");
 
-        store->buildPaths({ DerivedPath::Built { drvPath } });
+        store->buildPaths({
+            DerivedPath::Built {
+                .drvPath = drvPath,
+                .outputs = OutputsSpec::All { },
+            },
+        });
 
         auto outPathS = store->printStorePath(outPath);
 

src/nix/develop.cc

@@ -232,7 +232,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     auto shellDrvPath = writeDerivation(*evalStore, drv);
 
     /* Build the derivation. */
-    store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore);
+    store->buildPaths(
+        { DerivedPath::Built {
+            .drvPath = shellDrvPath,
+            .outputs = OutputsSpec::All { },
+        }},
+        bmNormal, evalStore);
 
     for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);

src/nix/flake.cc

@@ -513,8 +513,12 @@ struct CmdFlakeCheck : FlakeCommand
                                     auto drvPath = checkDerivation(
                                         fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
                                         *attr2.value, attr2.pos);
-                                    if (drvPath && attr_name == settings.thisSystem.get())
+                                    if (drvPath && attr_name == settings.thisSystem.get()) {
-                                        drvPaths.push_back(DerivedPath::Built{*drvPath});
+                                        drvPaths.push_back(DerivedPath::Built {
+                                            .drvPath = *drvPath,
+                                            .outputs = OutputsSpec::All { },
+                                        });
+                                    }
                                 }
                             }
                         }

src/nix/profile.cc

@@ -22,7 +22,7 @@ struct ProfileElementSource
     // FIXME: record original attrpath.
     FlakeRef resolvedRef;
     std::string attrPath;
-    OutputsSpec outputs;
+    ExtendedOutputsSpec outputs;
 
     bool operator < (const ProfileElementSource & other) const
     {
@@ -44,7 +44,7 @@ struct ProfileElement
     std::string describe() const
     {
         if (source)
-            return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs));
+            return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string());
         StringSet names;
         for (auto & path : storePaths)
             names.insert(DrvName(path.name()).name);
@@ -126,7 +126,7 @@ struct ProfileManifest
                         parseFlakeRef(e[sOriginalUrl]),
                         parseFlakeRef(e[sUrl]),
                         e["attrPath"],
-                        e["outputs"].get<OutputsSpec>()
+                        e["outputs"].get<ExtendedOutputsSpec>()
                     };
                 }
                 elements.emplace_back(std::move(element));
@@ -308,12 +308,12 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
             auto & [res, info] = builtPaths[installable.get()];
 
-            if (info.originalRef && info.resolvedRef && info.attrPath && info.outputsSpec) {
+            if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) {
                 element.source = ProfileElementSource {
                     .originalRef = *info.originalRef,
                     .resolvedRef = *info.resolvedRef,
                     .attrPath = *info.attrPath,
-                    .outputs = *info.outputsSpec,
+                    .outputs = *info.extendedOutputsSpec,
                 };
             }
 
@@ -497,7 +497,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                     .originalRef = installable->flakeRef,
                     .resolvedRef = *info.resolvedRef,
                     .attrPath = *info.attrPath,
-                    .outputs = installable->outputsSpec,
+                    .outputs = installable->extendedOutputsSpec,
                 };
 
                 installables.push_back(installable);
@@ -553,8 +553,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
             logger->cout("%d %s %s %s", i,
-                element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
+                element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
-                element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
+                element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
                 concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
         }
     }