From 3ec0c82fab94533807c5c3bb25df2b43d8339ed3 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Fri, 29 Mar 2019 16:18:25 +0100 Subject: [PATCH] Fixed dependency resolution --- src/libexpr/primops/flake.cc | 204 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 31 ++++-- src/nix/build.cc | 10 +- src/nix/flake.cc | 25 +++-- 4 files changed, 168 insertions(+), 102 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 70d1b871a..1ad9ad3f8 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -39,9 +39,8 @@ std::shared_ptr readRegistry(const Path & path) /* Write the registry or lock file to a file. */ void writeRegistry(FlakeRegistry registry, Path path) { - nlohmann::json json = {}; + nlohmann::json json; json["version"] = 1; - json["flakes"] = {}; for (auto elem : registry.entries) { json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; } @@ -49,6 +48,85 @@ void writeRegistry(FlakeRegistry registry, Path path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } +LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + + LockFile::FlakeEntry entry(flakeRef); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + for (auto & x : entry.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : entry.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + return json; +} + +void writeLockFile(LockFile lockFile, Path path) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeRequires"]; + for (auto & x : lockFile.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // '4' = indentation in json file +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -142,17 +220,13 @@ struct FlakeSourceInfo std::optional revCount; }; -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeRef directFlakeRef = FlakeRef(flakeRef); - if (!flakeRef.isDirect()) { - directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); - } - assert(directFlakeRef.isDirect()); - // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes.. + FlakeRef directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); if (auto refData = std::get_if(&directFlakeRef.data)) { - // FIXME: require hash in pure mode. + if (evalSettings.pureEval && !impureIsAllowed && !directFlakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", directFlakeRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -207,7 +281,8 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } -Flake getFlake(EvalState & state, const FlakeRef & flakeRef) +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", @@ -264,14 +339,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake lacks attribute 'provides'"); - auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack - - flake.lockFile = readRegistry(lockFile); - for (auto & entry : flake.lockFile->entries) - if (!entry.second.ref.isImmutable()) - throw Error("flake lock file '%s' contains mutable entry '%s'", - lockFile, entry.second.ref.to_string()); + const Path lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack + flake.lockFile = readLockFile(lockFile); return flake; } @@ -307,77 +377,50 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flake dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake = true) { - Dependencies deps; - std::queue todo; - bool isTopLevel = true; - todo.push(topRef); + Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); + Dependencies deps(flake); - auto registries = state.getFlakeRegistries(); + for (auto & nonFlakeInfo : flake.nonFlakeRequires) + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); - while (!todo.empty()) { - auto flakeRef = todo.front(); - todo.pop(); - - if (std::get_if(&flakeRef.data)) - flakeRef = lookupFlake(state, flakeRef, registries); - - if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) - throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); - - auto flake = getFlake(state, flakeRef); - - if (isTopLevel) { - deps.topFlakeId = flake.id; - isTopLevel = false; - } - - for (auto & flakeRef : flake.requires) - todo.push(flakeRef); - - for (auto & x : flake.nonFlakeRequires) - deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first)); - // TODO (Nick): If there are 2 non-flake dependencies with the same - // FlakeId, this will lead to trouble! One of the dependencies won't - // be used! - - deps.flakes.push_back(flake); - } + for (auto & newFlakeRef : flake.requires) + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, impureTopRef, false)); return deps; } -FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) +LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) { - FlakeRegistry newLockFile; - Dependencies deps = resolveFlake(evalState, flakeRef, false); - // Nick assumed that "topRefPure" means that the Flake for flakeRef can be - // fetched purely. - for (auto const& require : deps.flakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref); - // The FlakeRefs are immutable because they come out of the Flake objects. - if (require.id != deps.topFlakeId) - newLockFile.entries.insert_or_assign(require.id, entry); - // TODO (Nick): If there are 2 flake dependencies with the same FlakeId, - // one of them gets ignored! - } - for (auto const& nonFlake : deps.nonFlakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref); - newLockFile.entries.insert_or_assign(nonFlake.id, entry); - // We are assuming the sets of FlakeIds for flakes and non-flakes - // are disjoint. - } - return newLockFile; + LockFile::FlakeEntry entry(deps.flake.ref); + + for (Dependencies & deps : deps.flakeDeps) + entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); + + for (NonFlake & nonFlake : deps.nonFlakeDeps) + entry.nonFlakeEntries.insert_or_assign(nonFlake.id, nonFlake.ref); + + return entry; } -void updateLockFile(EvalState & state, Path path) +LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopRef) +{ + Dependencies deps = resolveFlake(evalState, flakeRef, impureTopRef); + LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); + LockFile lockFile; + lockFile.flakeEntries = entry.flakeEntries; + lockFile.nonFlakeEntries = entry.nonFlakeEntries; + return lockFile; +} + +void updateLockFile(EvalState & state, Path path, bool impureTopRef) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { - FlakeRegistry newLockFile = updateLockFile(state, flakeRef); - writeRegistry(newLockFile, path + "/flake.lock"); + LockFile lockFile = getLockFile(state, flakeRef, impureTopRef); + writeLockFile(lockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { @@ -393,19 +436,19 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) Dependencies deps = resolveFlake(state, flakeRef, impure); - // FIXME: we should call each flake with only its dependencies - // (rather than the closure of the top-level flake). + // // FIXME: we should call each flake with only its dependencies + // // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - state.mkAttrs(*vResult, dependencies.flakes.size()); + state.mkAttrs(*vResult, deps.flakeDeps.size()); Value * vTop = 0; - for (auto & flake : deps.flakes) { + for (auto & flake : deps.flakeDeps) { auto vFlake = state.allocAttr(*vResult, flake.id); - if (topFlakeId == flake.second.id) vTop = vFlake; + if (deps.topFlakeId == flake.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); @@ -431,6 +474,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) return vTop; } +// This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index a8f907784..019688f37 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -19,6 +19,20 @@ struct FlakeRegistry std::map entries; }; +struct LockFile +{ + struct FlakeEntry + { + FlakeRef ref; + std::map flakeEntries; + std::map nonFlakeEntries; + FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; + }; + + std::map flakeEntries; + std::map nonFlakeEntries; +}; + Path getUserRegistryPath(); Value * makeFlakeRegistryValue(EvalState & state); @@ -37,7 +51,7 @@ struct Flake Path path; std::optional revCount; std::vector requires; - std::shared_ptr lockFile; + LockFile lockFile; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc // date @@ -55,18 +69,17 @@ struct NonFlake NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; -Flake getFlake(EvalState &, const FlakeRef &); +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct Dependencies { - FlakeId topFlakeId; - std::vector flakes; - std::vector nonFlakes; + Flake flake; + std::vector flakeDeps; // The flake dependencies + std::vector nonFlakeDeps; + Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); -FlakeRegistry updateLockFile(EvalState &, Flake &); - -void updateLockFile(EvalState &, Path); +void updateLockFile(EvalState &, Path path, bool impureTopRef); } diff --git a/src/nix/build.cc b/src/nix/build.cc index f6908b0c0..226c21e9e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,11 +78,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - std::string flakeUri = ""; - if(updateLock) - for (uint i = 0; i < installables.size(); i++) - // if (auto flakeUri = installableToFlakeUri) - updateLockFile(*evalState, flakeUri); + // std::string flakeUri = ""; + // if(updateLock) + // for (uint i = 0; i < installables.size(); i++) + // // if (auto flakeUri = installableToFlakeUri) + // updateLockFile(*evalState, flakeUri, true); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 07d31c45a..ff291aa80 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -5,6 +5,7 @@ #include "progress-bar.hh" #include "eval.hh" #include +#include using namespace nix; @@ -80,13 +81,21 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, true, true); - for (auto & flake : deps.flakes) - printFlakeInfo(flake, json); + std::queue todo; + todo.push(deps); - for (auto & nonFlake : deps.nonFlakes) - printNonFlakeInfo(nonFlake, json); + while (!todo.empty()) { + deps = todo.front(); + todo.pop(); + + for (auto & nonFlake : deps.nonFlakeDeps) + printNonFlakeInfo(nonFlake, json); + + for (auto & newDeps : deps.flakeDeps) + todo.push(newDeps); + } } }; @@ -107,7 +116,7 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs auto evalState = std::make_shared(searchPath, store); if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath); + updateLockFile(*evalState, gitPath, true); } }; @@ -126,7 +135,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; @@ -220,7 +229,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs auto it = userRegistry.entries.find(flakeId); if (it != userRegistry.entries.end()) { FlakeRef oldRef = it->second.ref; - it->second.ref = getFlake(*evalState, oldRef).ref; + it->second.ref = getFlake(*evalState, oldRef, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else