Merge remote-tracking branch 'upstream/master' into drv-outputs-map-allow-missing

src/build-remote/build-remote.cc

@@ -170,7 +170,45 @@ static int _main(int argc, char * * argv)
                     if (rightType && !canBuildLocally)
                         std::cerr << "# postpone\n";
                     else
+                    {
+                        // build the hint template.
+                        string hintstring =  "derivation: %s\nrequired (system, features): (%s, %s)";
+                        hintstring += "\n%s available machines:";
+                        hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+                        for (unsigned int i = 0; i < machines.size(); ++i) {
+                          hintstring += "\n(%s, %s, %s, %s)";
+                        }
+
+                        // add the template values.
+                        string drvstr;
+                        if (drvPath.has_value())
+                            drvstr = drvPath->to_string();
+                        else
+                            drvstr = "<unknown>";
+
+                        auto hint = hintformat(hintstring);
+                        hint
+                          % drvstr
+                          % neededSystem
+                          % concatStringsSep<StringSet>(", ", requiredFeatures)
+                          % machines.size();
+
+                        for (auto & m : machines) {
+                          hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                            % m.maxJobs
+                            % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+                            % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
+                        }
+
+                        logError({
+                              .name = "Remote build",
+                              .description = "Failed to find a machine for remote build!",
+                              .hint = hint
+                        });
+
                         std::cerr << "# decline\n";
+                    }
                     break;
                 }
 

src/libexpr/eval-cache.cc

@@ -405,7 +405,7 @@ Value & AttrCursor::forceValue()
     return v;
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
 {
     if (root->db) {
         if (!cachedValue)
@@ -422,9 +422,12 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second))
+                    else if (std::get_if<failed_t>(&attr->second)) {
-                        throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                        if (forceErrors)
+                            debug("reevaluating failed cached attribute '%s'");
                         else
+                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    } else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -469,9 +472,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
 {
-    auto p = maybeGetAttr(name);
+    auto p = maybeGetAttr(name, forceErrors);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return p;
@@ -600,7 +603,7 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath);
+    auto aDrvPath = getAttr(root->state.sDrvPath, true);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has

src/libexpr/eval-cache.hh

@@ -9,6 +9,8 @@
 
 namespace nix::eval_cache {
 
+MakeError(CachedEvalError, EvalError);
+
 class AttrDb;
 class AttrCursor;
 
@@ -92,11 +94,11 @@ public:
 
     std::string getAttrPathStr(Symbol name) const;
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> getAttr(Symbol name);
+    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> getAttr(std::string_view name);
 

src/libexpr/eval.hh

@@ -375,6 +375,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
         "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+
+    Setting<bool> useEvalCache{this, true, "eval-cache",
+        "Whether to use the flake evaluation cache."};
 };
 
 extern EvalSettings evalSettings;

src/libstore/build.cc

@@ -4880,8 +4880,17 @@ void Worker::run(const Goals & _topGoals)
             waitForInput();
         else {
             if (awake.empty() && 0 == settings.maxBuildJobs)
+            {
+                if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds");
+                            "or enable remote builds."
+                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                else
+                   throw Error("unable to start any build; remote machines may not have "
+                            "all required system features."
+                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+
+            }
             assert(!awake.empty());
         }
     }

src/nix/app.cc

@@ -8,7 +8,7 @@ namespace nix {
 
 App Installable::toApp(EvalState & state)
 {
-    auto [cursor, attrPath] = getCursor(state, true);
+    auto [cursor, attrPath] = getCursor(state);
 
     auto type = cursor->getAttr("type")->getString();
 

src/nix/build.cc

@@ -64,17 +64,24 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 
         if (dryRun) return;
 
-        if (outLink != "") {
+        if (outLink != "")
-            for (size_t i = 0; i < buildables.size(); ++i) {
+            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                for (auto & output : buildables[i].outputs)
+                for (size_t i = 0; i < buildables.size(); ++i)
-                    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                    std::visit(overloaded {
+                        [&](BuildableOpaque bo) {
+                            std::string symlink = outLink;
+                            if (i) symlink += fmt("-%d", i);
+                            store2->addPermRoot(bo.path, absPath(symlink), true);
+                        },
+                        [&](BuildableFromDrv bfd) {
+                            for (auto & output : bfd.outputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);
                                 store2->addPermRoot(output.second, absPath(symlink), true);
                             }
-            }
+                        },
-        }
+                    }, buildables[i]);
 
         updateProfile(buildables);
     }

src/nix/command.cc

@@ -128,20 +128,25 @@ void MixProfile::updateProfile(const Buildables & buildables)
 {
     if (!profile) return;
 
-    std::optional<StorePath> result;
+    std::vector<StorePath> result;
 
     for (auto & buildable : buildables) {
-        for (auto & output : buildable.outputs) {
+        std::visit(overloaded {
-            if (result)
+            [&](BuildableOpaque bo) {
-                throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
+                result.push_back(bo.path);
-            result = output.second;
+            },
+            [&](BuildableFromDrv bfd) {
+                for (auto & output : bfd.outputs) {
+                    result.push_back(output.second);
                 }
+            },
+        }, buildable);
     }
 
-    if (!result)
+    if (result.size() != 1)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
+        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
-    updateProfile(*result);
+    updateProfile(result[0]);
 }
 
 MixDefaultProfile::MixDefaultProfile()

src/nix/flake.cc

@@ -572,7 +572,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
 
-        auto [cursor, attrPath] = installable.getCursor(*evalState, true);
+        auto [cursor, attrPath] = installable.getCursor(*evalState);
 
         auto templateDir = cursor->getAttr("path")->getString();
 
@@ -782,7 +782,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
 struct CmdFlakeShow : FlakeCommand
 {
     bool showLegacy = false;
-    bool useEvalCache = true;
 
     CmdFlakeShow()
     {
@@ -791,12 +790,6 @@ struct CmdFlakeShow : FlakeCommand
             .description = "show the contents of the 'legacyPackages' output",
             .handler = {&showLegacy, true}
         });
-
-        addFlag({
-            .longName = "no-eval-cache",
-            .description = "do not use the flake evaluation cache",
-            .handler = {[&]() { useEvalCache = false; }}
-        });
     }
 
     std::string description() override
@@ -934,7 +927,7 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto cache = openEvalCache(*state, flake, useEvalCache);
+        auto cache = openEvalCache(*state, flake);
 
         visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
     }

src/nix/installables.cc

@@ -183,8 +183,7 @@ void completeFlakeRefWithFragment(
             auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)),
+                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
-                true);
 
             auto root = evalCache->getRoot();
 
@@ -273,7 +272,7 @@ Buildable Installable::toBuildable()
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-Installable::getCursors(EvalState & state, bool useEvalCache)
+Installable::getCursors(EvalState & state)
 {
     auto evalCache =
         std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
@@ -282,9 +281,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache)
 }
 
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-Installable::getCursor(EvalState & state, bool useEvalCache)
+Installable::getCursor(EvalState & state)
 {
-    auto cursors = getCursors(state, useEvalCache);
+    auto cursors = getCursors(state);
     if (cursors.empty())
         throw Error("cannot find flake attribute '%s'", what());
     return cursors[0];
@@ -308,16 +307,15 @@ struct InstallableStorePath : Installable
             for (auto & [name, output] : drv.outputs)
                 outputs.emplace(name, output.path(*store, drv.name));
             return {
-                Buildable {
+                BuildableFromDrv {
                     .drvPath = storePath,
                     .outputs = std::move(outputs)
                 }
             };
         } else {
             return {
-                Buildable {
+                BuildableOpaque {
-                    .drvPath = {},
+                    .path = storePath,
-                    .outputs = {{"out", storePath}}
                 }
             };
         }
@@ -333,32 +331,19 @@ Buildables InstallableValue::toBuildables()
 {
     Buildables res;
 
-    StorePathSet drvPaths;
+    std::map<StorePath, OutputPathMap> drvsToOutputs;
 
+    // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
-        Buildable b{.drvPath = drv.drvPath};
-        drvPaths.insert(drv.drvPath);
-
         auto outputName = drv.outputName;
         if (outputName == "")
-            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
+            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
-
+        drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
-        b.outputs.emplace(outputName, drv.outPath);
-
-        res.push_back(std::move(b));
     }
 
-    // Hack to recognize .all: if all drvs have the same drvPath,
+    for (auto & i : drvsToOutputs)
-    // merge the buildables.
+        res.push_back(BuildableFromDrv { i.first, i.second });
-    if (drvPaths.size() == 1) {
+
-        Buildable b{.drvPath = *drvPaths.begin()};
-        for (auto & b2 : res)
-            for (auto & output : b2.outputs)
-                b.outputs.insert_or_assign(output.first, output.second);
-        Buildables bs;
-        bs.push_back(std::move(b));
-        return bs;
-    } else
     return res;
 }
 
@@ -434,12 +419,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
+    std::shared_ptr<flake::LockedFlake> lockedFlake)
-    bool useEvalCache)
 {
     auto fingerprint = lockedFlake->getFingerprint();
     return make_ref<nix::eval_cache::EvalCache>(
-        useEvalCache && evalSettings.pureEval
+        evalSettings.useEvalCache && evalSettings.pureEval
             ? std::optional { std::cref(fingerprint) }
             : std::nullopt,
         state,
@@ -474,10 +458,9 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
 
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
-
     auto lockedFlake = getLockedFlake();
 
-    auto cache = openEvalCache(*state, lockedFlake, true);
+    auto cache = openEvalCache(*state, lockedFlake);
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
@@ -531,11 +514,10 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
+InstallableFlake::getCursors(EvalState & state)
 {
     auto evalCache = openEvalCache(state,
-        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)),
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
-        useEvalCache);
 
     auto root = evalCache->getRoot();
 
@@ -656,14 +638,17 @@ Buildables build(ref<Store> store, Realise mode,
 
     for (auto & i : installables) {
         for (auto & b : i->toBuildables()) {
-            if (b.drvPath) {
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    pathsToBuild.push_back({bo.path});
+                },
+                [&](BuildableFromDrv bfd) {
                     StringSet outputNames;
-                for (auto & output : b.outputs)
+                    for (auto & output : bfd.outputs)
                         outputNames.insert(output.first);
-                pathsToBuild.push_back({*b.drvPath, outputNames});
+                    pathsToBuild.push_back({bfd.drvPath, outputNames});
-            } else
+                },
-                for (auto & output : b.outputs)
+            }, b);
-                    pathsToBuild.push_back({output.second});
             buildables.push_back(std::move(b));
         }
     }
@@ -684,16 +669,23 @@ StorePathSet toStorePaths(ref<Store> store,
 
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
-            for (auto & output : b.outputs)
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    outPaths.insert(bo.path);
+                },
+                [&](BuildableFromDrv bfd) {
+                    for (auto & output : bfd.outputs)
                         outPaths.insert(output.second);
+                },
+            }, b);
     } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
         for (auto & i : installables)
             for (auto & b : i->toBuildables())
-                if (b.drvPath)
+                if (auto bfd = std::get_if<BuildableFromDrv>(&b))
-                    outPaths.insert(*b.drvPath);
+                    outPaths.insert(bfd->drvPath);
     }
 
     return outPaths;
@@ -717,20 +709,21 @@ StorePathSet toDerivations(ref<Store> store,
     StorePathSet drvPaths;
 
     for (auto & i : installables)
-        for (auto & b : i->toBuildables()) {
+        for (auto & b : i->toBuildables())
-            if (!b.drvPath) {
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                for (auto & output : b.outputs) {
+                    auto derivers = store->queryValidDerivers(bo.path);
-                    auto derivers = store->queryValidDerivers(output.second);
                     if (derivers.empty())
                         throw Error("'%s' does not have a known deriver", i->what());
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
-                }
+                },
-            } else
+                [&](BuildableFromDrv bfd) {
-                drvPaths.insert(*b.drvPath);
+                    drvPaths.insert(bfd.drvPath);
-        }
+                },
+            }, b);
 
     return drvPaths;
 }

src/nix/installables.hh

@@ -14,12 +14,20 @@ struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
 
-struct Buildable
+struct BuildableOpaque {
-{
+    StorePath path;
-    std::optional<StorePath> drvPath;
+};
+
+struct BuildableFromDrv {
+    StorePath drvPath;
     std::map<std::string, StorePath> outputs;
 };
 
+typedef std::variant<
+    BuildableOpaque,
+    BuildableFromDrv
+> Buildable;
+
 typedef std::vector<Buildable> Buildables;
 
 struct App
@@ -54,10 +62,10 @@ struct Installable
     }
 
     virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache);
+    getCursors(EvalState & state);
 
     std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-    getCursor(EvalState & state, bool useEvalCache);
+    getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
@@ -110,7 +118,7 @@ struct InstallableFlake : InstallableValue
     std::pair<Value *, Pos> toValue(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache) override;
+    getCursors(EvalState & state) override;
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
@@ -119,7 +127,6 @@ struct InstallableFlake : InstallableValue
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
-    bool useEvalCache);
 
 }

src/nix/log.cc

@@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand
 
         RunPager pager;
         for (auto & sub : subs) {
-            auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr;
+            auto log = std::visit(overloaded {
-            for (auto & output : b.outputs) {
+                [&](BuildableOpaque bo) {
-                if (log) break;
+                    return sub->getBuildLog(bo.path);
-                log = sub->getBuildLog(output.second);
+                },
-            }
+                [&](BuildableFromDrv bfd) {
+                    return sub->getBuildLog(bfd.drvPath);
+                },
+            }, b);
             if (!log) continue;
             stopProgressBar();
             printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());

src/nix/repl.cc

@@ -33,12 +33,17 @@ extern "C" {
 #include "command.hh"
 #include "finally.hh"
 
+#if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
+#endif
 
 namespace nix {
 
-struct NixRepl : gc
+struct NixRepl
+    #if HAVE_BOEHMGC
+    : gc
+    #endif
 {
     string curDir;
     std::unique_ptr<EvalState> state;

src/nix/search.cc

@@ -177,7 +177,7 @@ struct CmdSearch : InstallableCommand, MixJSON
             }
         };
 
-        for (auto & [cursor, prefix] : installable->getCursors(*state, true))
+        for (auto & [cursor, prefix] : installable->getCursors(*state))
             visit(*cursor, parseAttrPath(*state, prefix));
 
         if (!json && !results)