lulu-berlin/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Improve lock file generation

Browse source 
This is now done in a single pass. Also fixes some issues when
updating flakes with circular dependencies. Finally, when using
'--recreate-lock-file --commit-lock-file', the commit message now
correctly shows the differences.
This commit is contained in:
Eelco Dolstra 2020-03-27 21:08:41 +01:00
parent 3fa1e7dace
commit 015f8f1c13
2 changed files with 189 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

204
src/libexpr/flake/flake.cc
View file

@ -306,49 +306,41 @@ LockedFlake lockFlake(
auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache);
LockFile oldLockFile;
if (!lockFlags.recreateLockFile) {
// FIXME: symlink attack // FIXME: symlink attack
oldLockFile = LockFile::read( auto oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
}
debug("old lock file: %s", oldLockFile); debug("old lock file: %s", oldLockFile);
LockFile newLockFile, prevLockFile;
std::vector<InputPath> prevUnresolved;
// FIXME: check whether all overrides are used. // FIXME: check whether all overrides are used.
std::map<InputPath, FlakeInput> overrides; std::map<InputPath, FlakeInput> overrides;
for (auto & i : lockFlags.inputOverrides) for (auto & i : lockFlags.inputOverrides)
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
/* Compute the new lock file. This is dones as a fixpoint LockFile newLockFile;
iteration: we repeat until the new lock file no longer changes
and there are no unresolved "follows" inputs. */
while (true) {
std::vector<InputPath> unresolved;
/* Recurse into the flake inputs. */
std::function<void(
const FlakeInputs & flakeInputs,
std::shared_ptr<const Node> oldLocks,
std::shared_ptr<Node> newLocks,
const InputPath & inputPathPrefix)>
updateLocks;
std::vector<FlakeRef> parents; std::vector<FlakeRef> parents;
std::map<InputPath, InputPath> follows;
updateLocks = [&]( std::function<void(
const FlakeInputs & flakeInputs, const FlakeInputs & flakeInputs,
std::shared_ptr<const Node> oldLocks, std::shared_ptr<Node> node,
std::shared_ptr<Node> newLocks, const InputPath & inputPathPrefix,
const InputPath & inputPathPrefix) std::shared_ptr<const Node> oldNode)>
computeLocks;
computeLocks = [&](
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)
{ {
debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix));
/* Get the overrides (i.e. attributes of the form /* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */ 'inputs.nixops.inputs.nixpkgs.url = ...'). */
// FIXME: check this
for (auto & [id, input] : flake.inputs) { for (auto & [id, input] : flake.inputs) {
for (auto & [idOverride, inputOverride] : input.overrides) { for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputPath(inputPathPrefix); auto inputPath(inputPathPrefix);
@ -365,59 +357,53 @@ LockedFlake lockFlake(
auto inputPath(inputPathPrefix); auto inputPath(inputPathPrefix);
inputPath.push_back(id); inputPath.push_back(id);
auto inputPathS = concatStringsSep("/", inputPath); auto inputPathS = concatStringsSep("/", inputPath);
debug("computing input '%s'", concatStringsSep("/", inputPath));
/* Do we have an override for this input from one of /* Do we have an override for this input from one of the
the ancestors? */ ancestors? */
auto i = overrides.find(inputPath); auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end(); bool hasOverride = i != overrides.end();
auto & input = hasOverride ? i->second : input2; auto & input = hasOverride ? i->second : input2;
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
if (input.follows) { if (input.follows) {
/* This is a "follows" input if (hasOverride)
(i.e. 'inputs.nixpkgs.follows = /* 'follows' from an override is relative to the
"dwarffs/nixpkgs"). Resolve the source and copy root of the graph. */
its inputs. Note that the source is normally follows.insert_or_assign(inputPath, *input.follows);
relative to the current node of the lock file else {
(e.g. "dwarffs/nixpkgs" refers to the nixpkgs /* Otherwise, it's relative to the current flake. */
input of the dwarffs input of the root flake), InputPath path(inputPathPrefix);
but if it's from an override, it's relative to for (auto & i : *input.follows) path.push_back(i);
the *root* of the lock file. */ follows.insert_or_assign(inputPath, path);
auto follows = (hasOverride ? newLockFile.root : newLocks)->findInput(*input.follows); }
if (follows)
newLocks->inputs.insert_or_assign(id, follows);
else
/* We haven't processed the source of the
"follows" yet (e.g. "dwarffs/nixpkgs"). So
we'll need another round of the fixpoint
iteration. */
// FIXME: now that LockFile is a graph, we
// could pre-create the missing node.
unresolved.push_back(inputPath);
continue; continue;
} }
/* Do we have an entry in the existing lock file? And /* Do we have an entry in the existing lock file? And we
we don't have a --update-input flag for this don't have a --update-input flag for this input? */
input? */
auto oldLockIt =
lockFlags.inputUpdates.count(inputPath)
? oldLocks->inputs.end()
: oldLocks->inputs.find(id);
std::shared_ptr<const LockedNode> oldLock; std::shared_ptr<const LockedNode> oldLock;
if (oldLockIt != oldLocks->inputs.end()) {
if (oldNode && !lockFlags.inputUpdates.count(inputPath)) {
auto oldLockIt = oldNode->inputs.find(id);
if (oldLockIt != oldNode->inputs.end())
oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second); oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second);
assert(oldLock);
} }
if (oldLock if (oldLock
&& oldLock->originalRef == input.ref && oldLock->originalRef == input.ref
&& !hasOverride) && !hasOverride)
{ {
/* Copy the input from the old lock file if its debug("keeping existing input '%s'", inputPathS);
flakeref didn't change and there is no override
from a higher level flake. */ /* Copy the input from the old lock since its flakeref
newLocks->inputs.insert_or_assign(id, std::make_shared<LockedNode>(*oldLock)); didn't change and there is no override from a
higher level flake. */
auto childNode = std::make_shared<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake);
node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input /* If we have an --update-input flag for an input
of this input, then we must fetch the flake to of this input, then we must fetch the flake to
@ -432,33 +418,35 @@ LockedFlake lockFlake(
if (hasChildUpdate) { if (hasChildUpdate) {
auto inputFlake = getFlake( auto inputFlake = getFlake(
state, oldLock->lockedRef, oldLock->info, false, flakeCache); state, oldLock->lockedRef, oldLock->info, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
updateLocks(inputFlake.inputs,
oldLock,
newLocks->inputs.find(id)->second,
inputPath);
} else { } else {
/* No need to fetch this flake, we can be /* No need to fetch this flake, we can be
lazy. However there may be new overrides on lazy. However there may be new overrides on the
the inputs of this flake, so we need to inputs of this flake, so we need to check
check those. */ those. */
FlakeInputs fakeInputs; FlakeInputs fakeInputs;
for (auto & i : oldLock->inputs) for (auto & i : oldLock->inputs) {
auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second);
// Note: this node is not locked in case
// of a circular reference back to the root.
if (lockedNode)
fakeInputs.emplace(i.first, FlakeInput { fakeInputs.emplace(i.first, FlakeInput {
.ref = std::dynamic_pointer_cast<LockedNode>(i.second)->originalRef .ref = lockedNode->originalRef
}); });
else {
InputPath path(inputPath);
path.push_back(i.first);
follows.insert_or_assign(path, InputPath());
}
}
updateLocks(fakeInputs, computeLocks(fakeInputs, childNode, inputPath, oldLock);
oldLock,
newLocks->inputs.find(id)->second,
inputPath);
} }
} else { } else {
/* We need to update/create a new lock file /* We need to create a new lock file entry. So fetch
entry. So fetch the flake/non-flake. */ this input. */
if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) if (!lockFlags.allowMutable && !input.ref.input->isImmutable())
throw Error("cannot update flake input '%s' in pure mode", inputPathS); throw Error("cannot update flake input '%s' in pure mode", inputPathS);
@ -466,13 +454,10 @@ LockedFlake lockFlake(
if (input.isFlake) { if (input.isFlake) {
auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache);
newLocks->inputs.insert_or_assign(id, auto childNode = std::make_shared<LockedNode>(
std::make_shared<LockedNode>(inputFlake.lockedRef, inputFlake.originalRef, inputFlake.sourceInfo->info)); inputFlake.lockedRef, inputFlake.originalRef, inputFlake.sourceInfo->info);
/* Recursively process the inputs of this node->inputs.insert_or_assign(id, childNode);
flake. Also, unless we already have this
flake in the top-level lock file, use this
flake's own lock file. */
/* Guard against circular flake imports. */ /* Guard against circular flake imports. */
for (auto & parent : parents) for (auto & parent : parents)
@ -481,42 +466,55 @@ LockedFlake lockFlake(
parents.push_back(input.ref); parents.push_back(input.ref);
Finally cleanup([&]() { parents.pop_back(); }); Finally cleanup([&]() { parents.pop_back(); });
updateLocks(inputFlake.inputs, /* Recursively process the inputs of this
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
own lock file. */
computeLocks(
inputFlake.inputs, childNode, inputPath,
oldLock oldLock
? std::dynamic_pointer_cast<const Node>(oldLock) ? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read( : LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
newLocks->inputs.find(id)->second,
inputPath);
} }
else { else {
auto [sourceInfo, lockedRef] = fetchOrSubstituteTree( auto [sourceInfo, lockedRef] = fetchOrSubstituteTree(
state, input.ref, {}, lockFlags.useRegistries, flakeCache); state, input.ref, {}, lockFlags.useRegistries, flakeCache);
newLocks->inputs.insert_or_assign(id, node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false)); std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false));
} }
} }
} }
}; };
updateLocks(flake.inputs, oldLockFile.root, newLockFile.root, {}); computeLocks(
flake.inputs, newLockFile.root, {},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
/* Check if there is a cycle in the "follows" inputs. */ /* Insert edges for 'follows' overrides. */
if (!unresolved.empty() && unresolved == prevUnresolved) { for (auto & [from, to] : follows) {
std::vector<std::string> ss; debug("adding 'follows' node from '%s' to '%s'",
for (auto & i : unresolved) concatStringsSep("/", from),
ss.push_back(concatStringsSep("/", i)); concatStringsSep("/", to));
throw Error("cycle or missing input detected in flake inputs: %s", concatStringsSep(", ", ss));
assert(!from.empty());
InputPath fromParent(from);
fromParent.pop_back();
auto fromParentNode = newLockFile.root->findInput(fromParent);
assert(fromParentNode);
auto toNode = newLockFile.root->findInput(to);
if (!toNode)
throw Error("flake input '%s' follows non-existent flake input '%s'",
concatStringsSep("/", from),
concatStringsSep("/", to));
fromParentNode->inputs.insert_or_assign(from.back(), toNode);
} }
std::swap(unresolved, prevUnresolved);
/* Done with the fixpoint iteration? */
if (newLockFile == prevLockFile) break;
prevLockFile = newLockFile;
};
debug("new lock file: %s", newLockFile); debug("new lock file: %s", newLockFile);
/* Check whether we need to / can write the new lock file. */ /* Check whether we need to / can write the new lock file. */

2
tests/flakes.sh
View file

@ -518,7 +518,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF EOF
nix flake update $flake3Dir nix flake update $flake3Dir
[[ $(jq .nodes.foo.locked $flake3Dir/flake.lock) = $(jq .nodes.bar.locked $flake3Dir/flake.lock) ]] [[ $(jq .nodes.root.inputs.foo $flake3Dir/flake.lock) = $(jq .nodes.root.inputs.bar $flake3Dir/flake.lock) ]]
cat > $flake3Dir/flake.nix <<EOF cat > $flake3Dir/flake.nix <<EOF
{ {