lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Merge pull request #7484 from edolstra/fix-7417

Browse source 
InstallableFlake::toDerivedPaths(): Support paths and store paths
This commit is contained in:
Eelco Dolstra 2023-01-10 15:57:58 +01:00 committed by GitHub
parent cc02e11967 59cc920cc0
commit 1c98daf6e8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 263 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

202
src/libcmd/installables.cc
View file

@ -358,7 +358,7 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
}
}
DerivedPath Installable::toDerivedPath()
DerivedPathWithInfo Installable::toDerivedPath()
{
auto buildables = toDerivedPaths();
if (buildables.size() != 1)
@ -422,21 +422,9 @@ struct InstallableStorePath : Installable
return req.to_string(*store);
}
DerivedPaths toDerivedPaths() override
DerivedPathsWithInfo toDerivedPaths() override
{
return { req };
}
StorePathSet toDrvPaths(ref<Store> store) override
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> StorePathSet {
return { bfd.drvPath };
},
[&](const DerivedPath::Opaque & bo) -> StorePathSet {
return { getDeriver(store, *this, bo.path) };
},
}, req.raw());
return {{.path = req, .info = {} }};
}
std::optional<StorePath> getStorePath() override
@ -452,34 +440,6 @@ struct InstallableStorePath : Installable
}
};
DerivedPaths InstallableValue::toDerivedPaths()
{
DerivedPaths res;
std::map<StorePath, std::set<std::string>> drvsToOutputs;
RealisedPath::Set drvsToCopy;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
for (auto & outputName : drv.outputsToInstall)
drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath);
}
for (auto & i : drvsToOutputs)
res.push_back(DerivedPath::Built { i.first, i.second });
return res;
}
StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
{
StorePathSet res;
for (auto & drv : toDerivations())
res.insert(drv.drvPath);
return res;
}
struct InstallableAttrPath : InstallableValue
{
SourceExprCommand & cmd;
@ -509,40 +469,45 @@ struct InstallableAttrPath : InstallableValue
return {vRes, pos};
}
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
};
DerivedPathsWithInfo toDerivedPaths() override
{
auto v = toValue(*state).first;
std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
{
auto v = toValue(*state).first;
Bindings & autoArgs = *cmd.getAutoArgs(*state);
Bindings & autoArgs = *cmd.getAutoArgs(*state);
DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false);
DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false);
// Backward compatibility hack: group results by drvPath. This
// helps keep .all output together.
std::map<StorePath, DerivedPath::Built> byDrvPath;
std::vector<DerivationInfo> res;
for (auto & drvInfo : drvInfos) {
auto drvPath = drvInfo.queryDrvPath();
if (!drvPath)
throw Error("'%s' is not a derivation", what());
for (auto & drvInfo : drvInfos) {
auto drvPath = drvInfo.queryDrvPath();
if (!drvPath)
throw Error("'%s' is not a derivation", what());
std::set<std::string> outputsToInstall;
std::set<std::string> outputsToInstall;
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
res.push_back(DerivationInfo {
.drvPath = *drvPath,
.outputsToInstall = std::move(outputsToInstall)
});
auto derivedPath = byDrvPath.emplace(*drvPath, DerivedPath::Built { .drvPath = *drvPath }).first;
for (auto & output : outputsToInstall)
derivedPath->second.outputs.insert(output);
}
DerivedPathsWithInfo res;
for (auto & [_, info] : byDrvPath)
res.push_back({ .path = { info } });
return res;
}
return res;
}
};
std::vector<std::string> InstallableFlake::getActualAttrPaths()
{
@ -630,7 +595,7 @@ InstallableFlake::InstallableFlake(
throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
}
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
@ -638,8 +603,38 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto attrPath = attr->getAttrPathStr();
if (!attr->isDerivation())
throw Error("flake output attribute '%s' is not a derivation", attrPath);
if (!attr->isDerivation()) {
// FIXME: use eval cache?
auto v = attr->forceValue();
if (v.type() == nPath) {
PathSet context;
auto storePath = state->copyPathToStore(context, Path(v.path));
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),
}
}};
}
else if (v.type() == nString) {
PathSet context;
auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
auto storePath = state->store->maybeParseStorePath(s);
if (storePath && context.count(std::string(s))) {
return {{
.path = DerivedPath::Opaque {
.path = std::move(*storePath),
}
}};
} else
throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
}
else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
}
auto drvPath = attr->forceDerivation();
@ -674,20 +669,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo {
.drvPath = std::move(drvPath),
.outputsToInstall = std::move(outputsToInstall),
.priority = priority,
};
return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
}
std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
{
std::vector<DerivationInfo> res;
res.push_back(std::get<2>(toDerivation()));
return res;
return {{
.path = DerivedPath::Built {
.drvPath = std::move(drvPath),
.outputs = std::move(outputsToInstall),
},
.info = {
.priority = priority,
.originalRef = flakeRef,
.resolvedRef = getLockedFlake()->flake.lockedRef,
.attrPath = attrPath,
.outputsSpec = outputsSpec,
}
}};
}
std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
@ -895,13 +889,19 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
struct Aux
{
ExtraPathInfo info;
std::shared_ptr<Installable> installable;
};
std::vector<DerivedPath> pathsToBuild;
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
std::map<DerivedPath, std::vector<Aux>> backmap;
for (auto & i : installables) {
for (auto b : i->toDerivedPaths()) {
pathsToBuild.push_back(b);
backmap[b].push_back(i);
pathsToBuild.push_back(b.path);
backmap[b.path].push_back({.info = b.info, .installable = i});
}
}
@ -914,7 +914,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
printMissing(store, pathsToBuild, lvlError);
for (auto & path : pathsToBuild) {
for (auto & installable : backmap[path]) {
for (auto & aux : backmap[path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
OutputPathMap outputs;
@ -943,10 +943,14 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
output, *drvOutput->second);
}
}
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}});
res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info}});
},
[&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}});
res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info}});
},
}, path.raw());
}
@ -962,16 +966,22 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (!buildResult.success())
buildResult.rethrow();
for (auto & installable : backmap[buildResult.path]) {
for (auto & aux : backmap[buildResult.path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}});
res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info,
.result = buildResult}});
},
[&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}});
res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info,
.result = buildResult}});
},
}, buildResult.path.raw());
}
@ -1056,7 +1066,7 @@ StorePathSet Installable::toDerivations(
[&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath);
},
}, b.raw());
}, b.path.raw());
return drvPaths;
}

47
src/libcmd/installables.hh
View file

@ -52,26 +52,42 @@ enum class OperateOn {
Derivation
};
struct ExtraPathInfo
{
std::optional<NixInt> priority;
std::optional<FlakeRef> originalRef;
std::optional<FlakeRef> resolvedRef;
std::optional<std::string> attrPath;
// FIXME: merge with DerivedPath's 'outputs' field?
std::optional<OutputsSpec> outputsSpec;
};
/* A derived path with any additional info that commands might
need from the derivation. */
struct DerivedPathWithInfo
{
DerivedPath path;
ExtraPathInfo info;
};
struct BuiltPathWithResult
{
BuiltPath path;
ExtraPathInfo info;
std::optional<BuildResult> result;
};
typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo;
struct Installable
{
virtual ~Installable() { }
virtual std::string what() const = 0;
virtual DerivedPaths toDerivedPaths() = 0;
virtual DerivedPathsWithInfo toDerivedPaths() = 0;
virtual StorePathSet toDrvPaths(ref<Store> store)
{
throw Error("'%s' cannot be converted to a derivation path", what());
}
DerivedPath toDerivedPath();
DerivedPathWithInfo toDerivedPath();
UnresolvedApp toApp(EvalState & state);
@ -146,19 +162,6 @@ struct InstallableValue : Installable
ref<EvalState> state;
InstallableValue(ref<EvalState> state) : state(state) {}
struct DerivationInfo
{
StorePath drvPath;
std::set<std::string> outputsToInstall;
std::optional<NixInt> priority;
};
virtual std::vector<DerivationInfo> toDerivations() = 0;
DerivedPaths toDerivedPaths() override;
StorePathSet toDrvPaths(ref<Store> store) override;
};
struct InstallableFlake : InstallableValue
@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue
Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
std::vector<DerivationInfo> toDerivations() override;
DerivedPathsWithInfo toDerivedPaths() override;
std::pair<Value *, PosIdx> toValue(EvalState & state) override;

28
src/libexpr/eval.cc
View file

@ -2166,7 +2166,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet
if (canonicalizePath)
path = canonPath(*path);
if (copyToStore)
path = copyPathToStore(context, std::move(path).toOwned());
path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned()));
return path;
}
@ -2215,26 +2215,26 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet
}
std::string EvalState::copyPathToStore(PathSet & context, const Path & path)
StorePath EvalState::copyPathToStore(PathSet & context, const Path & path)
{
if (nix::isDerivation(path))
error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>();
Path dstPath;
auto i = srcToStore.find(path);
if (i != srcToStore.end())
dstPath = store->printStorePath(i->second);
else {
auto p = settings.readOnlyMode
auto dstPath = [&]() -> StorePath
{
auto i = srcToStore.find(path);
if (i != srcToStore.end()) return i->second;
auto dstPath = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
allowPath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
}
allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
return dstPath;
}();
context.insert(dstPath);
context.insert(store->printStorePath(dstPath));
return dstPath;
}

2
src/libexpr/eval.hh
View file

@ -379,7 +379,7 @@ public:
bool canonicalizePath = true,
std::string_view errorCtx = "");
std::string copyPathToStore(PathSet & context, const Path & path);
StorePath copyPathToStore(PathSet & context, const Path & path);
/* Path coercion. Converts strings, paths and derivations to a
path. The result is guaranteed to be a canonicalised, absolute

3
src/libexpr/value-to-json.cc
View file

@ -1,6 +1,7 @@
#include "value-to-json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include "store-api.hh"
#include <cstdlib>
#include <iomanip>
@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict,
case nPath:
if (copyToStore)
out = state.copyPathToStore(context, v.path);
out = state.store->printStorePath(state.copyPathToStore(context, v.path));
else
out = v.path;
break;

5
src/nix/app.cc
View file

@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable
{
}
std::string what() const override { return derivedPath.to_string(*store); }
DerivedPaths toDerivedPaths() override
DerivedPathsWithInfo toDerivedPaths() override
{
return {derivedPath};
return {{derivedPath}};
}
std::optional<StorePath> getStorePath() override

10
src/nix/build.cc
View file

@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
if (dryRun) {
std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
auto b = i->toDerivedPaths();
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
}
for (auto & i : installables)
for (auto & b : i->toDerivedPaths())
pathsToBuild.push_back(b.path);
printMissing(store, pathsToBuild, lvlError);
if (json)
logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump());
return;
}

2
src/nix/log.cc
View file

@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand
[&](const DerivedPath::Built & bfd) {
return logSub.getBuildLog(bfd.drvPath);
},
}, b.raw());
}, b.path.raw());
if (!log) continue;
stopProgressBar();
printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri());

68
src/nix/profile.cc
View file

@ -32,12 +32,14 @@ struct ProfileElementSource
}
};
const int defaultPriority = 5;
struct ProfileElement
{
StorePathSet storePaths;
std::optional<ProfileElementSource> source;
bool active = true;
int priority = 5;
int priority = defaultPriority;
std::string describe() const
{
@ -251,13 +253,20 @@ struct ProfileManifest
}
};
static std::map<Installable *, BuiltPaths>
static std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>>
builtPathsPerInstallable(
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
{
std::map<Installable *, BuiltPaths> res;
for (auto & [installable, builtPath] : builtPaths)
res[installable.get()].push_back(builtPath.path);
std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> res;
for (auto & [installable, builtPath] : builtPaths) {
auto & r = res[installable.get()];
/* Note that there could be conflicting info
(e.g. meta.priority fields) if the installable returned
multiple derivations. So pick one arbitrarily. FIXME:
print a warning? */
r.first.push_back(builtPath.path);
r.second = builtPath.info;
}
return res;
}
@ -297,28 +306,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
for (auto & installable : installables) {
ProfileElement element;
auto & [res, info] = builtPaths[installable.get()];
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
// FIXME: make build() return this?
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
if (info.originalRef && info.resolvedRef && info.attrPath && info.outputsSpec) {
element.source = ProfileElementSource {
installable2->flakeRef,
resolvedRef,
attrPath,
installable2->outputsSpec
.originalRef = *info.originalRef,
.resolvedRef = *info.resolvedRef,
.attrPath = *info.attrPath,
.outputs = *info.outputsSpec,
};
if(drv.priority) {
element.priority = *drv.priority;
}
}
if(priority) { // if --priority was specified we want to override the priority of the installable
element.priority = *priority;
};
// If --priority was specified we want to override the
// priority of the installable.
element.priority =
priority
? *priority
: info.priority.value_or(defaultPriority);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
element.updateStorePaths(getEvalStore(), store, res);
manifest.elements.push_back(std::move(element));
}
@ -476,18 +482,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
Strings{},
lockFlags);
auto [attrPath, resolvedRef, drv] = installable->toDerivation();
auto derivedPaths = installable->toDerivedPaths();
if (derivedPaths.empty()) continue;
auto & info = derivedPaths[0].info;
if (element.source->resolvedRef == resolvedRef) continue;
assert(info.resolvedRef && info.attrPath);
if (element.source->resolvedRef == info.resolvedRef) continue;
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.source->attrPath, element.source->resolvedRef, *info.resolvedRef);
element.source = ProfileElementSource {
installable->flakeRef,
resolvedRef,
attrPath,
installable->outputsSpec
.originalRef = installable->flakeRef,
.resolvedRef = *info.resolvedRef,
.attrPath = *info.attrPath,
.outputs = installable->outputsSpec,
};
installables.push_back(installable);
@ -515,7 +525,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)];
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first);
}
updateProfile(manifest.build(store));

8
src/nix/store-copy-log.cc
View file

@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
auto dstStore = getDstStore();
auto & dstLogStore = require<LogStore>(*dstStore);
StorePathSet drvPaths;
for (auto & i : installables)
for (auto & drvPath : i->toDrvPaths(getEvalStore()))
drvPaths.insert(drvPath);
for (auto & drvPath : drvPaths) {
for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) {
if (auto log = srcLogStore.getBuildLog(drvPath))
dstLogStore.addBuildLog(drvPath, *log);
else

66
tests/flakes/build-paths.sh Normal file
View file

@ -0,0 +1,66 @@
source ./common.sh
flake1Dir=$TEST_ROOT/flake1
flake2Dir=$TEST_ROOT/flake2
mkdir -p $flake1Dir $flake2Dir
writeSimpleFlake $flake2Dir
tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2
hash=$(nix hash path $flake2Dir)
dep=$(nix store add-path ./common.sh)
cat > $flake1Dir/flake.nix <<EOF
{
inputs.flake2.url = "file://$TEST_ROOT/flake.tar.gz";
outputs = { self, flake2 }: {
a1 = builtins.fetchTarball {
#type = "tarball";
url = "file://$TEST_ROOT/flake.tar.gz";
sha256 = "$hash";
};
a2 = ./foo;
a3 = ./.;
a4 = self.outPath;
# FIXME
a5 = self;
a6 = flake2.outPath;
# FIXME
a7 = "${flake2}/config.nix";
# This is only allowed in impure mode.
a8 = builtins.storePath $dep;
a9 = "$dep";
};
}
EOF
echo bar > $flake1Dir/foo
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1
[[ -e $TEST_ROOT/result/simple.nix ]]
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2
[[ $(cat $TEST_ROOT/result) = bar ]]
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6
[[ -e $TEST_ROOT/result/simple.nix ]]
nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8
diff common.sh $TEST_ROOT/result
(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9)

1
tests/local.mk
View file

@ -9,6 +9,7 @@ nix_tests = \
flakes/check.sh \
flakes/unlocked-override.sh \
flakes/absolute-paths.sh \
flakes/build-paths.sh \
ca/gc.sh \
gc.sh \
remote-store.sh \