From 845fc3f605be6dd1140ab81eefb969da1cc5346b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2022 22:09:32 +0100 Subject: [PATCH 1/8] Merge toDerivations() into toDerivedPaths() toDerivedPaths() now returns DerivedPathWithInfo, which is DerivedPath with some attributes needed by 'nix profile' etc. Preparation for #7417. --- src/libcmd/installables.cc | 175 +++++++++++++++++-------------------- src/libcmd/installables.hh | 47 +++++----- src/nix/app.cc | 5 +- src/nix/build.cc | 10 ++- src/nix/log.cc | 2 +- src/nix/profile.cc | 67 ++++++++------ src/nix/store-copy-log.cc | 8 +- src/nix/why-depends.cc | 2 +- 8 files changed, 154 insertions(+), 162 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5cdd3e12c..97bad5c45 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -358,7 +358,7 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -422,21 +422,9 @@ struct InstallableStorePath : Installable return req.to_string(*store); } - DerivedPaths toDerivedPaths() override + DerivedPathsWithInfo toDerivedPaths() override { - return { req }; - } - - StorePathSet toDrvPaths(ref store) override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> StorePathSet { - return { bfd.drvPath }; - }, - [&](const DerivedPath::Opaque & bo) -> StorePathSet { - return { getDeriver(store, *this, bo.path) }; - }, - }, req.raw()); + return {{req}}; } std::optional getStorePath() override @@ -452,34 +440,6 @@ struct InstallableStorePath : Installable } }; -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - for (auto & outputName : drv.outputsToInstall) - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; @@ -509,40 +469,52 @@ struct InstallableAttrPath : InstallableValue return {vRes, pos}; } - virtual std::vector toDerivations() override; -}; + DerivedPathsWithInfo toDerivedPaths() override + { + auto v = toValue(*state).first; -std::vector InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; + Bindings & autoArgs = *cmd.getAutoArgs(*state); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); + DerivedPathsWithInfo res; - std::vector res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map byDrvPath; - std::set outputsToInstall; + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); - if (auto outputNames = std::get_if(&outputsSpec)) - outputsToInstall = *outputNames; - else - for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) - outputsToInstall.insert(output.first); + std::set outputsToInstall; - res.push_back(DerivationInfo { - .drvPath = *drvPath, - .outputsToInstall = std::move(outputsToInstall) - }); + if (auto outputNames = std::get_if(&outputsSpec)) + outputsToInstall = *outputNames; + else + for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) + outputsToInstall.insert(output.first); + + auto i = byDrvPath.find(*drvPath); + if (i == byDrvPath.end()) { + byDrvPath[*drvPath] = res.size(); + res.push_back({ + .path = DerivedPath::Built { + .drvPath = std::move(*drvPath), + .outputs = std::move(outputsToInstall), + } + }); + } else { + for (auto & output : outputsToInstall) + std::get(res[i->second].path).outputs.insert(output); + } + } + + return res; } - - return res; -} +}; std::vector InstallableFlake::getActualAttrPaths() { @@ -630,7 +602,7 @@ InstallableFlake::InstallableFlake( throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); } -std::tuple InstallableFlake::toDerivation() +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() { Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); @@ -674,20 +646,19 @@ std::tuple InstallableF if (auto outputNames = std::get_if(&outputsSpec)) outputsToInstall = *outputNames; - auto drvInfo = DerivationInfo { - .drvPath = std::move(drvPath), - .outputsToInstall = std::move(outputsToInstall), - .priority = priority, - }; - - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; -} - -std::vector InstallableFlake::toDerivations() -{ - std::vector res; - res.push_back(std::get<2>(toDerivation())); - return res; + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::move(outputsToInstall), + }, + .info = { + .priority = priority, + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + .attrPath = attrPath, + .outputsSpec = outputsSpec, + } + }}; } std::pair InstallableFlake::toValue(EvalState & state) @@ -895,13 +866,19 @@ std::vector, BuiltPathWithResult>> Instal if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ExtraInfo info; + std::shared_ptr installable; + }; + std::vector pathsToBuild; - std::map>> backmap; + std::map> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } @@ -914,7 +891,7 @@ std::vector, BuiltPathWithResult>> Instal printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { OutputPathMap outputs; @@ -946,10 +923,14 @@ std::vector, BuiltPathWithResult>> Instal output, *drvOutput->second); } } - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -965,16 +946,22 @@ std::vector, BuiltPathWithResult>> Instal if (!buildResult.success()) buildResult.rethrow(); - for (auto & installable : backmap[buildResult.path]) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; for (auto & path : buildResult.builtOutputs) outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -1059,7 +1046,7 @@ StorePathSet Installable::toDerivations( [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 02ea351d3..95ab4a40e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -52,26 +52,42 @@ enum class OperateOn { Derivation }; +struct ExtraInfo +{ + std::optional priority; + std::optional originalRef; + std::optional resolvedRef; + std::optional attrPath; + // FIXME: merge with DerivedPath's 'outputs' field? + std::optional outputsSpec; +}; + +/* A derived path with any additional info that commands might + need from the derivation. */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ExtraInfo info; +}; + struct BuiltPathWithResult { BuiltPath path; + ExtraInfo info; std::optional result; }; +typedef std::vector DerivedPathsWithInfo; + struct Installable { virtual ~Installable() { } virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; + virtual DerivedPathsWithInfo toDerivedPaths() = 0; - virtual StorePathSet toDrvPaths(ref store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } - - DerivedPath toDerivedPath(); + DerivedPathWithInfo toDerivedPath(); UnresolvedApp toApp(EvalState & state); @@ -146,19 +162,6 @@ struct InstallableValue : Installable ref state; InstallableValue(ref state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::set outputsToInstall; - std::optional priority; - }; - - virtual std::vector toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref store) override; }; struct InstallableFlake : InstallableValue @@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple toDerivation(); - - std::vector toDerivations() override; + DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; diff --git a/src/nix/app.cc b/src/nix/app.cc index 5658f2a52..a8d7e115b 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable { } - std::string what() const override { return derivedPath.to_string(*store); } - DerivedPaths toDerivedPaths() override + DerivedPathsWithInfo toDerivedPaths() override { - return {derivedPath}; + return {{derivedPath}}; } std::optional getStorePath() override diff --git a/src/nix/build.cc b/src/nix/build.cc index 94b169167..12b22d999 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (dryRun) { std::vector pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; } diff --git a/src/nix/log.cc b/src/nix/log.cc index 72d02ef11..a0598ca13 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand [&](const DerivedPath::Built & bfd) { return logSub.getBuildLog(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 11910523d..db702db1b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -32,12 +32,14 @@ struct ProfileElementSource } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional source; bool active = true; - int priority = 5; + int priority = defaultPriority; std::string describe() const { @@ -251,13 +253,19 @@ struct ProfileManifest } }; -static std::map +static std::map> builtPathsPerInstallable( const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath.path); + std::map> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res[installable.get()]; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } @@ -297,28 +305,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + auto & [res, info] = builtPaths[installable.get()]; - - if (auto installable2 = std::dynamic_pointer_cast(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + if (info.originalRef && info.resolvedRef && info.attrPath && info.outputsSpec) { element.source = ProfileElementSource { - installable2->flakeRef, - resolvedRef, - attrPath, - installable2->outputsSpec + .originalRef = *info.originalRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = *info.outputsSpec, }; - - if(drv.priority) { - element.priority = *drv.priority; - } } - if(priority) { // if --priority was specified we want to override the priority of the installable - element.priority = *priority; - }; + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : info.priority.value_or(defaultPriority); - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } @@ -476,18 +481,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto & info = derivedPaths[0].info; - if (element.source->resolvedRef == resolvedRef) continue; + assert(info.resolvedRef && info.attrPath); + + if (element.source->resolvedRef == info.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); + element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); element.source = ProfileElementSource { - installable->flakeRef, - resolvedRef, - attrPath, - installable->outputsSpec + .originalRef = installable->flakeRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = installable->outputsSpec, }; installables.push_back(installable); @@ -515,7 +524,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first); } updateProfile(manifest.build(store)); diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 2e288f743..d5fab5f2f 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand auto dstStore = getDstStore(); auto & dstLogStore = require(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 723017497..661df965e 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -111,7 +111,7 @@ struct CmdWhyDepends : SourceExprCommand } return maybePath->second; }, - }, derivedDependency.raw()); + }, derivedDependency.path.raw()); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); From bda879170fbf8ff8c5397948ccc0e1695d23871a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 20 Dec 2022 14:58:39 +0100 Subject: [PATCH 2/8] EvalState::copyPathToStore(): Return a StorePath --- src/libexpr/eval.cc | 28 ++++++++++++++-------------- src/libexpr/eval.hh | 2 +- src/libexpr/value-to-json.cc | 3 ++- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 084ccbee2..29d109ac4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2250,7 +2250,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (canonicalizePath) path = canonPath(*path); if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); + path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); return path; } @@ -2293,26 +2293,26 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) throwEvalError("file names are not allowed to end in '%1%'", drvExtension); - Path dstPath; - auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode + auto dstPath = [&]() -> StorePath + { + auto i = srcToStore.find(path); + if (i != srcToStore.end()) return i->second; + + auto dstPath = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); - context.insert(dstPath); + context.insert(store->printStorePath(dstPath)); return dstPath; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 21666339b..52b1736fe 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,7 +400,7 @@ public: bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5dc453b2e..c35c876e3 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" #include "util.hh" +#include "store-api.hh" #include #include @@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.copyPathToStore(context, v.path); + out = state.store->printStorePath(state.copyPathToStore(context, v.path)); else out = v.path; break; From 5c97b5a3988c7dd28e617734c2eba669ee0c1288 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2022 23:01:15 +0100 Subject: [PATCH 3/8] InstallableFlake::toDerivedPaths(): Support paths and store paths This makes 'nix build' work on paths (which will be copied to the store) and store paths (returned as is). E.g. the following flake output attributes can be built using 'nix build .#foo': foo = ./src; foo = self.outPath; foo = builtins.fetchTarball { ... }; foo = (builtins.fetchTree { .. }).outPath; foo = builtins.fetchTree { .. } + "/README.md"; foo = builtins.storePath /nix/store/...; Note that this is potentially risky, e.g. foo = /.; will cause Nix to try to copy the entire file system to the store. What doesn't work yet: foo = self; foo = builtins.fetchTree { .. }; because we don't handle attrsets with an outPath attribute in it yet, and foo = builtins.storePath /nix/store/.../README.md; since result symlinks have to point to a store path currently (rather than a file inside a store path). Fixes #7417. --- src/libcmd/installables.cc | 34 +++++++++++++++++-- tests/flakes/build-paths.sh | 66 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 tests/flakes/build-paths.sh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 97bad5c45..f4486bc2f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -610,8 +610,38 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto attrPath = attr->getAttrPathStr(); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + } + }}; + } + + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + } + }}; + } else + throw Error("flake output attribute '%s' evaluates to a string that does not denote a store path", attrPath); + } + + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } auto drvPath = attr->forceDerivation(); diff --git a/tests/flakes/build-paths.sh b/tests/flakes/build-paths.sh new file mode 100644 index 000000000..08b4d1763 --- /dev/null +++ b/tests/flakes/build-paths.sh @@ -0,0 +1,66 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +mkdir -p $flake1Dir $flake2Dir + +writeSimpleFlake $flake2Dir +tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2 +hash=$(nix hash path $flake2Dir) + +dep=$(nix store add-path ./common.sh) + +cat > $flake1Dir/flake.nix < $flake1Dir/foo + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 +[[ $(cat $TEST_ROOT/result) = bar ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 +diff common.sh $TEST_ROOT/result + +(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9) diff --git a/tests/local.mk b/tests/local.mk index 2f7f76261..55913e977 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -9,6 +9,7 @@ nix_tests = \ flakes/check.sh \ flakes/unlocked-override.sh \ flakes/absolute-paths.sh \ + flakes/build-paths.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From b80e4b57dae78490ed644a2e2a840dae39d1da4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:52:49 +0100 Subject: [PATCH 4/8] ExtraInfo -> ExtraPathInfo --- src/libcmd/installables.cc | 2 +- src/libcmd/installables.hh | 6 +++--- src/nix/profile.cc | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f5a436fbe..280212379 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -898,7 +898,7 @@ std::vector, BuiltPathWithResult>> Instal struct Aux { - ExtraInfo info; + ExtraPathInfo info; std::shared_ptr installable; }; diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 95ab4a40e..250cf7e83 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -52,7 +52,7 @@ enum class OperateOn { Derivation }; -struct ExtraInfo +struct ExtraPathInfo { std::optional priority; std::optional originalRef; @@ -67,13 +67,13 @@ struct ExtraInfo struct DerivedPathWithInfo { DerivedPath path; - ExtraInfo info; + ExtraPathInfo info; }; struct BuiltPathWithResult { BuiltPath path; - ExtraInfo info; + ExtraPathInfo info; std::optional result; }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index db702db1b..3da47568c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -253,11 +253,11 @@ struct ProfileManifest } }; -static std::map> +static std::map> builtPathsPerInstallable( const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map> res; + std::map> res; for (auto & [installable, builtPath] : builtPaths) { auto & r = res[installable.get()]; /* Note that there could be conflicting info From b4dc68f0be07edc3511a4ef276de1cb4e13a676d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:56:03 +0100 Subject: [PATCH 5/8] Show string in error message --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 280212379..d7fdbb13d 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -636,7 +636,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } }}; } else - throw Error("flake output attribute '%s' evaluates to a string that does not denote a store path", attrPath); + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); } else From 1123c42f9016c5df7ade9d917d5e1900af6688e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 14:57:35 +0100 Subject: [PATCH 6/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d7fdbb13d..409afd762 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -424,7 +424,7 @@ struct InstallableStorePath : Installable DerivedPathsWithInfo toDerivedPaths() override { - return {{req}}; + return {{.path = req, .info = {} }}; } std::optional getStorePath() override From 7f1af270ddffa1cd6d18c167bcebe56b93bfd127 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 15:08:46 +0100 Subject: [PATCH 7/8] Clean up toDerivedPaths() logic --- src/libcmd/installables.cc | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 409afd762..c0db2a715 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -478,11 +478,9 @@ struct InstallableAttrPath : InstallableValue DrvInfos drvInfos; getDerivations(*state, *v, "", autoArgs, drvInfos, false); - DerivedPathsWithInfo res; - // Backward compatibility hack: group results by drvPath. This // helps keep .all output together. - std::map byDrvPath; + std::map byDrvPath; for (auto & drvInfo : drvInfos) { auto drvPath = drvInfo.queryDrvPath(); @@ -497,21 +495,16 @@ struct InstallableAttrPath : InstallableValue for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) outputsToInstall.insert(output.first); - auto i = byDrvPath.find(*drvPath); - if (i == byDrvPath.end()) { - byDrvPath[*drvPath] = res.size(); - res.push_back({ - .path = DerivedPath::Built { - .drvPath = std::move(*drvPath), - .outputs = std::move(outputsToInstall), - } - }); - } else { - for (auto & output : outputsToInstall) - std::get(res[i->second].path).outputs.insert(output); - } + auto derivedPath = byDrvPath.emplace(*drvPath, DerivedPath::Built { .drvPath = *drvPath }).first; + + for (auto & output : outputsToInstall) + derivedPath->second.outputs.insert(output); } + DerivedPathsWithInfo res; + for (auto & [_, info] : byDrvPath) + res.push_back({ .path = { info } }); + return res; } }; From 59cc920cc0751f93d4a3b717da72505a7bab2ee7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jan 2023 15:20:30 +0100 Subject: [PATCH 8/8] Add a FIXME --- src/nix/profile.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 3da47568c..22ee51ab9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -262,7 +262,8 @@ builtPathsPerInstallable( auto & r = res[installable.get()]; /* Note that there could be conflicting info (e.g. meta.priority fields) if the installable returned - multiple derivations. So pick one arbitrarily. */ + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ r.first.push_back(builtPath.path); r.second = builtPath.info; }