diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 3579d8fff..5247cefa6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -103,7 +103,7 @@ static int _main(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + auto canBuildLocally = amWilling && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(requiredFeatures); @@ -170,7 +170,45 @@ static int _main(int argc, char * * argv) if (rightType && !canBuildLocally) std::cerr << "# postpone\n"; else + { + // build the hint template. + string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)"; + hintstring += "\n%s available machines:"; + hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; + + for (unsigned int i = 0; i < machines.size(); ++i) { + hintstring += "\n(%s, %s, %s, %s)"; + } + + // add the template values. + string drvstr; + if (drvPath.has_value()) + drvstr = drvPath->to_string(); + else + drvstr = ""; + + auto hint = hintformat(hintstring); + hint + % drvstr + % neededSystem + % concatStringsSep(", ", requiredFeatures) + % machines.size(); + + for (auto & m : machines) { + hint % concatStringsSep>(", ", m.systemTypes) + % m.maxJobs + % concatStringsSep(", ", m.supportedFeatures) + % concatStringsSep(", ", m.mandatoryFeatures); + } + + logError({ + .name = "Remote build", + .description = "Failed to find a machine for remote build!", + .hint = hint + }); + std::cerr << "# decline\n"; + } break; } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index deb32484f..46177a0a4 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -405,7 +405,7 @@ Value & AttrCursor::forceValue() return v; } -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { if (!cachedValue) @@ -422,9 +422,12 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) if (attr) { if (std::get_if(&attr->second)) return nullptr; - else if (std::get_if(&attr->second)) - throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - else + else if (std::get_if(&attr->second)) { + if (forceErrors) + debug("reevaluating failed cached attribute '%s'"); + else + throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + } else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } @@ -469,9 +472,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -std::shared_ptr AttrCursor::getAttr(Symbol name) +std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors) { - auto p = maybeGetAttr(name); + auto p = maybeGetAttr(name, forceErrors); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return p; @@ -600,7 +603,7 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath); + auto aDrvPath = getAttr(root->state.sDrvPath, true); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index afee85fa9..8ffffc0ed 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -9,6 +9,8 @@ namespace nix::eval_cache { +MakeError(CachedEvalError, EvalError); + class AttrDb; class AttrCursor; @@ -92,11 +94,11 @@ public: std::string getAttrPathStr(Symbol name) const; - std::shared_ptr maybeGetAttr(Symbol name); + std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); std::shared_ptr maybeGetAttr(std::string_view name); - std::shared_ptr getAttr(Symbol name); + std::shared_ptr getAttr(Symbol name, bool forceErrors = false); std::shared_ptr getAttr(std::string_view name); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0382298b3..5855b4ef2 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -375,6 +375,9 @@ struct EvalSettings : Config Setting traceFunctionCalls{this, false, "trace-function-calls", "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; }; extern EvalSettings evalSettings; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 233648465..b47aeff3b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4880,8 +4880,17 @@ void Worker::run(const Goals & _topGoals) waitForInput(); else { if (awake.empty() && 0 == settings.maxBuildJobs) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + { + if (getMachines().empty()) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + else + throw Error("unable to start any build; remote machines may not have " + "all required system features." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + + } assert(!awake.empty()); } } diff --git a/src/nix/app.cc b/src/nix/app.cc index 3935297cf..80acbf658 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -8,7 +8,7 @@ namespace nix { App Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state, true); + auto [cursor, attrPath] = getCursor(state); auto type = cursor->getAttr("type")->getString(); diff --git a/src/nix/build.cc b/src/nix/build.cc index 613cc15eb..13d14a7fb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -64,17 +64,24 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile if (dryRun) return; - if (outLink != "") { - for (size_t i = 0; i < buildables.size(); ++i) { - for (auto & output : buildables[i].outputs) - if (auto store2 = store.dynamic_pointer_cast()) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink), true); - } - } - } + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + for (size_t i = 0; i < buildables.size(); ++i) + std::visit(overloaded { + [&](BuildableOpaque bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2->addPermRoot(bo.path, absPath(symlink), true); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink), true); + } + }, + }, buildables[i]); updateProfile(buildables); } diff --git a/src/nix/command.cc b/src/nix/command.cc index af36dda89..da32819da 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -128,20 +128,25 @@ void MixProfile::updateProfile(const Buildables & buildables) { if (!profile) return; - std::optional result; + std::vector result; for (auto & buildable : buildables) { - for (auto & output : buildable.outputs) { - if (result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; - } + std::visit(overloaded { + [&](BuildableOpaque bo) { + result.push_back(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + result.push_back(output.second); + } + }, + }, buildable); } - if (!result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + if (result.size() != 1) + throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(*result); + updateProfile(result[0]); } MixDefaultProfile::MixDefaultProfile() diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 80d8654bc..653f8db1b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -572,7 +572,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand Strings{templateName == "" ? "defaultTemplate" : templateName}, Strings(attrsPathPrefixes), lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState, true); + auto [cursor, attrPath] = installable.getCursor(*evalState); auto templateDir = cursor->getAttr("path")->getString(); @@ -782,7 +782,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand { bool showLegacy = false; - bool useEvalCache = true; CmdFlakeShow() { @@ -791,12 +790,6 @@ struct CmdFlakeShow : FlakeCommand .description = "show the contents of the 'legacyPackages' output", .handler = {&showLegacy, true} }); - - addFlag({ - .longName = "no-eval-cache", - .description = "do not use the flake evaluation cache", - .handler = {[&]() { useEvalCache = false; }} - }); } std::string description() override @@ -934,7 +927,7 @@ struct CmdFlakeShow : FlakeCommand } }; - auto cache = openEvalCache(*state, flake, useEvalCache); + auto cache = openEvalCache(*state, flake); visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 59b52ce95..ea152709f 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -183,8 +183,7 @@ void completeFlakeRefWithFragment( auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags)), - true); + std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -273,7 +272,7 @@ Buildable Installable::toBuildable() } std::vector, std::string>> -Installable::getCursors(EvalState & state, bool useEvalCache) +Installable::getCursors(EvalState & state) { auto evalCache = std::make_shared(std::nullopt, state, @@ -282,9 +281,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache) } std::pair, std::string> -Installable::getCursor(EvalState & state, bool useEvalCache) +Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state, useEvalCache); + auto cursors = getCursors(state); if (cursors.empty()) throw Error("cannot find flake attribute '%s'", what()); return cursors[0]; @@ -308,16 +307,15 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputs) outputs.emplace(name, output.path(*store, drv.name)); return { - Buildable { + BuildableFromDrv { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - Buildable { - .drvPath = {}, - .outputs = {{"out", storePath}} + BuildableOpaque { + .path = storePath, } }; } @@ -333,33 +331,20 @@ Buildables InstallableValue::toBuildables() { Buildables res; - StorePathSet drvPaths; + std::map drvsToOutputs; + // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { - Buildable b{.drvPath = drv.drvPath}; - drvPaths.insert(drv.drvPath); - auto outputName = drv.outputName; if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - - b.outputs.emplace(outputName, drv.outPath); - - res.push_back(std::move(b)); + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); + drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); } - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = *drvPaths.begin()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; + for (auto & i : drvsToOutputs) + res.push_back(BuildableFromDrv { i.first, i.second }); + + return res; } struct InstallableAttrPath : InstallableValue @@ -434,12 +419,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked ref openEvalCache( EvalState & state, - std::shared_ptr lockedFlake, - bool useEvalCache) + std::shared_ptr lockedFlake) { auto fingerprint = lockedFlake->getFingerprint(); return make_ref( - useEvalCache && evalSettings.pureEval + evalSettings.useEvalCache && evalSettings.pureEval ? std::optional { std::cref(fingerprint) } : std::nullopt, state, @@ -474,10 +458,9 @@ static std::string showAttrPaths(const std::vector & paths) std::tuple InstallableFlake::toDerivation() { - auto lockedFlake = getLockedFlake(); - auto cache = openEvalCache(*state, lockedFlake, true); + auto cache = openEvalCache(*state, lockedFlake); auto root = cache->getRoot(); for (auto & attrPath : getActualAttrPaths()) { @@ -531,11 +514,10 @@ std::pair InstallableFlake::toValue(EvalState & state) } std::vector, std::string>> -InstallableFlake::getCursors(EvalState & state, bool useEvalCache) +InstallableFlake::getCursors(EvalState & state) { auto evalCache = openEvalCache(state, - std::make_shared(lockFlake(state, flakeRef, lockFlags)), - useEvalCache); + std::make_shared(lockFlake(state, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -656,14 +638,17 @@ Buildables build(ref store, Realise mode, for (auto & i : installables) { for (auto & b : i->toBuildables()) { - if (b.drvPath) { - StringSet outputNames; - for (auto & output : b.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back({*b.drvPath, outputNames}); - } else - for (auto & output : b.outputs) - pathsToBuild.push_back({output.second}); + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path}); + }, + [&](BuildableFromDrv bfd) { + StringSet outputNames; + for (auto & output : bfd.outputs) + outputNames.insert(output.first); + pathsToBuild.push_back({bfd.drvPath, outputNames}); + }, + }, b); buildables.push_back(std::move(b)); } } @@ -684,16 +669,23 @@ StorePathSet toStorePaths(ref store, if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); + std::visit(overloaded { + [&](BuildableOpaque bo) { + outPaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) + outPaths.insert(output.second); + }, + }, b); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; for (auto & i : installables) for (auto & b : i->toBuildables()) - if (b.drvPath) - outPaths.insert(*b.drvPath); + if (auto bfd = std::get_if(&b)) + outPaths.insert(bfd->drvPath); } return outPaths; @@ -717,20 +709,21 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) { - if (!b.drvPath) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - for (auto & output : b.outputs) { - auto derivers = store->queryValidDerivers(output.second); + for (auto & b : i->toBuildables()) + std::visit(overloaded { + [&](BuildableOpaque bo) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", i->what()); + auto derivers = store->queryValidDerivers(bo.path); if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); - } - } else - drvPaths.insert(*b.drvPath); - } + }, + [&](BuildableFromDrv bfd) { + drvPaths.insert(bfd.drvPath); + }, + }, b); return drvPaths; } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index eb34365d4..26e87ee3a 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -14,12 +14,20 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct Buildable -{ - std::optional drvPath; +struct BuildableOpaque { + StorePath path; +}; + +struct BuildableFromDrv { + StorePath drvPath; std::map outputs; }; +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + typedef std::vector Buildables; struct App @@ -54,10 +62,10 @@ struct Installable } virtual std::vector, std::string>> - getCursors(EvalState & state, bool useEvalCache); + getCursors(EvalState & state); std::pair, std::string> - getCursor(EvalState & state, bool useEvalCache); + getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const { @@ -110,7 +118,7 @@ struct InstallableFlake : InstallableValue std::pair toValue(EvalState & state) override; std::vector, std::string>> - getCursors(EvalState & state, bool useEvalCache) override; + getCursors(EvalState & state) override; std::shared_ptr getLockedFlake() const; @@ -119,7 +127,6 @@ struct InstallableFlake : InstallableValue ref openEvalCache( EvalState & state, - std::shared_ptr lockedFlake, - bool useEvalCache); + std::shared_ptr lockedFlake); } diff --git a/src/nix/log.cc b/src/nix/log.cc index 7e10d373a..33380dcf5 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { - auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; - for (auto & output : b.outputs) { - if (log) break; - log = sub->getBuildLog(output.second); - } + auto log = std::visit(overloaded { + [&](BuildableOpaque bo) { + return sub->getBuildLog(bo.path); + }, + [&](BuildableFromDrv bfd) { + return sub->getBuildLog(bfd.drvPath); + }, + }, b); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fb9050d0d..c3c9e54a8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -33,12 +33,17 @@ extern "C" { #include "command.hh" #include "finally.hh" +#if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include +#endif namespace nix { -struct NixRepl : gc +struct NixRepl + #if HAVE_BOEHMGC + : gc + #endif { string curDir; std::unique_ptr state; diff --git a/src/nix/search.cc b/src/nix/search.cc index 65a1e1818..430979274 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -177,7 +177,7 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state, true)) + for (auto & [cursor, prefix] : installable->getCursors(*state)) visit(*cursor, parseAttrPath(*state, prefix)); if (!json && !results)