Merge toDerivations() into toDerivedPaths()

toDerivedPaths() now returns DerivedPathWithInfo, which is DerivedPath
with some attributes needed by 'nix profile' etc.

Preparation for #7417.
This commit is contained in:
Eelco Dolstra 2022-12-15 22:09:32 +01:00
parent 17373debe9
commit 845fc3f605
8 changed files with 154 additions and 162 deletions

View file

@ -358,7 +358,7 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
} }
} }
DerivedPath Installable::toDerivedPath() DerivedPathWithInfo Installable::toDerivedPath()
{ {
auto buildables = toDerivedPaths(); auto buildables = toDerivedPaths();
if (buildables.size() != 1) if (buildables.size() != 1)
@ -422,21 +422,9 @@ struct InstallableStorePath : Installable
return req.to_string(*store); return req.to_string(*store);
} }
DerivedPaths toDerivedPaths() override DerivedPathsWithInfo toDerivedPaths() override
{ {
return { req }; return {{req}};
}
StorePathSet toDrvPaths(ref<Store> store) override
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> StorePathSet {
return { bfd.drvPath };
},
[&](const DerivedPath::Opaque & bo) -> StorePathSet {
return { getDeriver(store, *this, bo.path) };
},
}, req.raw());
} }
std::optional<StorePath> getStorePath() override std::optional<StorePath> getStorePath() override
@ -452,34 +440,6 @@ struct InstallableStorePath : Installable
} }
}; };
DerivedPaths InstallableValue::toDerivedPaths()
{
DerivedPaths res;
std::map<StorePath, std::set<std::string>> drvsToOutputs;
RealisedPath::Set drvsToCopy;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
for (auto & outputName : drv.outputsToInstall)
drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath);
}
for (auto & i : drvsToOutputs)
res.push_back(DerivedPath::Built { i.first, i.second });
return res;
}
StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
{
StorePathSet res;
for (auto & drv : toDerivations())
res.insert(drv.drvPath);
return res;
}
struct InstallableAttrPath : InstallableValue struct InstallableAttrPath : InstallableValue
{ {
SourceExprCommand & cmd; SourceExprCommand & cmd;
@ -509,11 +469,8 @@ struct InstallableAttrPath : InstallableValue
return {vRes, pos}; return {vRes, pos};
} }
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; DerivedPathsWithInfo toDerivedPaths() override
}; {
std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
{
auto v = toValue(*state).first; auto v = toValue(*state).first;
Bindings & autoArgs = *cmd.getAutoArgs(*state); Bindings & autoArgs = *cmd.getAutoArgs(*state);
@ -521,7 +478,12 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
DrvInfos drvInfos; DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false); getDerivations(*state, *v, "", autoArgs, drvInfos, false);
std::vector<DerivationInfo> res; DerivedPathsWithInfo res;
// Backward compatibility hack: group results by drvPath. This
// helps keep .all output together.
std::map<StorePath, size_t> byDrvPath;
for (auto & drvInfo : drvInfos) { for (auto & drvInfo : drvInfos) {
auto drvPath = drvInfo.queryDrvPath(); auto drvPath = drvInfo.queryDrvPath();
if (!drvPath) if (!drvPath)
@ -535,14 +497,24 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec))) for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first); outputsToInstall.insert(output.first);
res.push_back(DerivationInfo { auto i = byDrvPath.find(*drvPath);
.drvPath = *drvPath, if (i == byDrvPath.end()) {
.outputsToInstall = std::move(outputsToInstall) byDrvPath[*drvPath] = res.size();
res.push_back({
.path = DerivedPath::Built {
.drvPath = std::move(*drvPath),
.outputs = std::move(outputsToInstall),
}
}); });
} else {
for (auto & output : outputsToInstall)
std::get<DerivedPath::Built>(res[i->second].path).outputs.insert(output);
}
} }
return res; return res;
} }
};
std::vector<std::string> InstallableFlake::getActualAttrPaths() std::vector<std::string> InstallableFlake::getActualAttrPaths()
{ {
@ -630,7 +602,7 @@ InstallableFlake::InstallableFlake(
throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
} }
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
@ -674,20 +646,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames; outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo { return {{
.path = DerivedPath::Built {
.drvPath = std::move(drvPath), .drvPath = std::move(drvPath),
.outputsToInstall = std::move(outputsToInstall), .outputs = std::move(outputsToInstall),
},
.info = {
.priority = priority, .priority = priority,
}; .originalRef = flakeRef,
.resolvedRef = getLockedFlake()->flake.lockedRef,
return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; .attrPath = attrPath,
} .outputsSpec = outputsSpec,
}
std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() }};
{
std::vector<DerivationInfo> res;
res.push_back(std::get<2>(toDerivation()));
return res;
} }
std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
@ -895,13 +866,19 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (mode == Realise::Nothing) if (mode == Realise::Nothing)
settings.readOnlyMode = true; settings.readOnlyMode = true;
struct Aux
{
ExtraInfo info;
std::shared_ptr<Installable> installable;
};
std::vector<DerivedPath> pathsToBuild; std::vector<DerivedPath> pathsToBuild;
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; std::map<DerivedPath, std::vector<Aux>> backmap;
for (auto & i : installables) { for (auto & i : installables) {
for (auto b : i->toDerivedPaths()) { for (auto b : i->toDerivedPaths()) {
pathsToBuild.push_back(b); pathsToBuild.push_back(b.path);
backmap[b].push_back(i); backmap[b.path].push_back({.info = b.info, .installable = i});
} }
} }
@ -914,7 +891,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
printMissing(store, pathsToBuild, lvlError); printMissing(store, pathsToBuild, lvlError);
for (auto & path : pathsToBuild) { for (auto & path : pathsToBuild) {
for (auto & installable : backmap[path]) { for (auto & aux : backmap[path]) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
OutputPathMap outputs; OutputPathMap outputs;
@ -946,10 +923,14 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
output, *drvOutput->second); output, *drvOutput->second);
} }
} }
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info}});
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info}});
}, },
}, path.raw()); }, path.raw());
} }
@ -965,16 +946,22 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (!buildResult.success()) if (!buildResult.success())
buildResult.rethrow(); buildResult.rethrow();
for (auto & installable : backmap[buildResult.path]) { for (auto & aux : backmap[buildResult.path]) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs) for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath); outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info,
.result = buildResult}});
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info,
.result = buildResult}});
}, },
}, buildResult.path.raw()); }, buildResult.path.raw());
} }
@ -1059,7 +1046,7 @@ StorePathSet Installable::toDerivations(
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath); drvPaths.insert(bfd.drvPath);
}, },
}, b.raw()); }, b.path.raw());
return drvPaths; return drvPaths;
} }

View file

@ -52,26 +52,42 @@ enum class OperateOn {
Derivation Derivation
}; };
struct ExtraInfo
{
std::optional<NixInt> priority;
std::optional<FlakeRef> originalRef;
std::optional<FlakeRef> resolvedRef;
std::optional<std::string> attrPath;
// FIXME: merge with DerivedPath's 'outputs' field?
std::optional<OutputsSpec> outputsSpec;
};
/* A derived path with any additional info that commands might
need from the derivation. */
struct DerivedPathWithInfo
{
DerivedPath path;
ExtraInfo info;
};
struct BuiltPathWithResult struct BuiltPathWithResult
{ {
BuiltPath path; BuiltPath path;
ExtraInfo info;
std::optional<BuildResult> result; std::optional<BuildResult> result;
}; };
typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo;
struct Installable struct Installable
{ {
virtual ~Installable() { } virtual ~Installable() { }
virtual std::string what() const = 0; virtual std::string what() const = 0;
virtual DerivedPaths toDerivedPaths() = 0; virtual DerivedPathsWithInfo toDerivedPaths() = 0;
virtual StorePathSet toDrvPaths(ref<Store> store) DerivedPathWithInfo toDerivedPath();
{
throw Error("'%s' cannot be converted to a derivation path", what());
}
DerivedPath toDerivedPath();
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
@ -146,19 +162,6 @@ struct InstallableValue : Installable
ref<EvalState> state; ref<EvalState> state;
InstallableValue(ref<EvalState> state) : state(state) {} InstallableValue(ref<EvalState> state) : state(state) {}
struct DerivationInfo
{
StorePath drvPath;
std::set<std::string> outputsToInstall;
std::optional<NixInt> priority;
};
virtual std::vector<DerivationInfo> toDerivations() = 0;
DerivedPaths toDerivedPaths() override;
StorePathSet toDrvPaths(ref<Store> store) override;
}; };
struct InstallableFlake : InstallableValue struct InstallableFlake : InstallableValue
@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue
Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation(); DerivedPathsWithInfo toDerivedPaths() override;
std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, PosIdx> toValue(EvalState & state) override; std::pair<Value *, PosIdx> toValue(EvalState & state) override;

View file

@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable
{ {
} }
std::string what() const override { return derivedPath.to_string(*store); } std::string what() const override { return derivedPath.to_string(*store); }
DerivedPaths toDerivedPaths() override DerivedPathsWithInfo toDerivedPaths() override
{ {
return {derivedPath}; return {{derivedPath}};
} }
std::optional<StorePath> getStorePath() override std::optional<StorePath> getStorePath() override

View file

@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
if (dryRun) { if (dryRun) {
std::vector<DerivedPath> pathsToBuild; std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) { for (auto & i : installables)
auto b = i->toDerivedPaths(); for (auto & b : i->toDerivedPaths())
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); pathsToBuild.push_back(b.path);
}
printMissing(store, pathsToBuild, lvlError); printMissing(store, pathsToBuild, lvlError);
if (json) if (json)
logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump());
return; return;
} }

View file

@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
return logSub.getBuildLog(bfd.drvPath); return logSub.getBuildLog(bfd.drvPath);
}, },
}, b.raw()); }, b.path.raw());
if (!log) continue; if (!log) continue;
stopProgressBar(); stopProgressBar();
printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri());

View file

@ -32,12 +32,14 @@ struct ProfileElementSource
} }
}; };
const int defaultPriority = 5;
struct ProfileElement struct ProfileElement
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::optional<ProfileElementSource> source; std::optional<ProfileElementSource> source;
bool active = true; bool active = true;
int priority = 5; int priority = defaultPriority;
std::string describe() const std::string describe() const
{ {
@ -251,13 +253,19 @@ struct ProfileManifest
} }
}; };
static std::map<Installable *, BuiltPaths> static std::map<Installable *, std::pair<BuiltPaths, ExtraInfo>>
builtPathsPerInstallable( builtPathsPerInstallable(
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths) const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
{ {
std::map<Installable *, BuiltPaths> res; std::map<Installable *, std::pair<BuiltPaths, ExtraInfo>> res;
for (auto & [installable, builtPath] : builtPaths) for (auto & [installable, builtPath] : builtPaths) {
res[installable.get()].push_back(builtPath.path); auto & r = res[installable.get()];
/* Note that there could be conflicting info
(e.g. meta.priority fields) if the installable returned
multiple derivations. So pick one arbitrarily. */
r.first.push_back(builtPath.path);
r.second = builtPath.info;
}
return res; return res;
} }
@ -297,28 +305,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
for (auto & installable : installables) { for (auto & installable : installables) {
ProfileElement element; ProfileElement element;
auto & [res, info] = builtPaths[installable.get()];
if (info.originalRef && info.resolvedRef && info.attrPath && info.outputsSpec) {
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
// FIXME: make build() return this?
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
element.source = ProfileElementSource { element.source = ProfileElementSource {
installable2->flakeRef, .originalRef = *info.originalRef,
resolvedRef, .resolvedRef = *info.resolvedRef,
attrPath, .attrPath = *info.attrPath,
installable2->outputsSpec .outputs = *info.outputsSpec,
}; };
if(drv.priority) {
element.priority = *drv.priority;
}
} }
if(priority) { // if --priority was specified we want to override the priority of the installable // If --priority was specified we want to override the
element.priority = *priority; // priority of the installable.
}; element.priority =
priority
? *priority
: info.priority.value_or(defaultPriority);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); element.updateStorePaths(getEvalStore(), store, res);
manifest.elements.push_back(std::move(element)); manifest.elements.push_back(std::move(element));
} }
@ -476,18 +481,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
Strings{}, Strings{},
lockFlags); lockFlags);
auto [attrPath, resolvedRef, drv] = installable->toDerivation(); auto derivedPaths = installable->toDerivedPaths();
if (derivedPaths.empty()) continue;
auto & info = derivedPaths[0].info;
if (element.source->resolvedRef == resolvedRef) continue; assert(info.resolvedRef && info.attrPath);
if (element.source->resolvedRef == info.resolvedRef) continue;
printInfo("upgrading '%s' from flake '%s' to '%s'", printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef); element.source->attrPath, element.source->resolvedRef, *info.resolvedRef);
element.source = ProfileElementSource { element.source = ProfileElementSource {
installable->flakeRef, .originalRef = installable->flakeRef,
resolvedRef, .resolvedRef = *info.resolvedRef,
attrPath, .attrPath = *info.attrPath,
installable->outputsSpec .outputs = installable->outputsSpec,
}; };
installables.push_back(installable); installables.push_back(installable);
@ -515,7 +524,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
for (size_t i = 0; i < installables.size(); ++i) { for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i); auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)]; auto & element = manifest.elements[indices.at(i)];
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first);
} }
updateProfile(manifest.build(store)); updateProfile(manifest.build(store));

View file

@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
auto dstStore = getDstStore(); auto dstStore = getDstStore();
auto & dstLogStore = require<LogStore>(*dstStore); auto & dstLogStore = require<LogStore>(*dstStore);
StorePathSet drvPaths; for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) {
for (auto & i : installables)
for (auto & drvPath : i->toDrvPaths(getEvalStore()))
drvPaths.insert(drvPath);
for (auto & drvPath : drvPaths) {
if (auto log = srcLogStore.getBuildLog(drvPath)) if (auto log = srcLogStore.getBuildLog(drvPath))
dstLogStore.addBuildLog(drvPath, *log); dstLogStore.addBuildLog(drvPath, *log);
else else

View file

@ -111,7 +111,7 @@ struct CmdWhyDepends : SourceExprCommand
} }
return maybePath->second; return maybePath->second;
}, },
}, derivedDependency.raw()); }, derivedDependency.path.raw());
StorePathSet closure; StorePathSet closure;
store->computeFSClosure({packagePath}, closure, false, false); store->computeFSClosure({packagePath}, closure, false, false);