From 18c019b616f457b1f9a39da8cafc012be5ddffcc Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 1/6] Added nonFlakeRequires and the command `nix flake deps` --- src/libexpr/primops/flake.cc | 191 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 23 ++++- src/nix/build.cc | 14 +-- src/nix/flake.cc | 65 ++++++++++-- 4 files changed, 200 insertions(+), 93 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 7cfb2038c..c4ae29022 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -18,20 +18,19 @@ std::shared_ptr readRegistry(const Path & path) { auto registry = std::make_shared(); - try { - auto json = nlohmann::json::parse(readFile(path)); + if (!pathExists(path)) + return std::make_shared(); - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); + auto json = nlohmann::json::parse(readFile(path)); - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - registry->entries.emplace(i.key(), entry); - } - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; + registry->entries.emplace(i.key(), entry); } return registry; @@ -54,7 +53,6 @@ Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; } - std::shared_ptr getGlobalRegistry() { // FIXME: get from nixos.org. @@ -76,12 +74,20 @@ std::shared_ptr getFlagRegistry() const std::vector> EvalState::getFlakeRegistries() { std::vector> registries; - registries.push_back(getGlobalRegistry()); - registries.push_back(getUserRegistry()); + if (evalSettings.pureEval) { + registries.push_back(std::make_shared()); // global + registries.push_back(std::make_shared()); // user + registries.push_back(std::make_shared()); // local + } else { + registries.push_back(getGlobalRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getLocalRegistry()); + } registries.push_back(getFlagRegistry()); return registries; } +// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`. Value * makeFlakeRegistryValue(EvalState & state) { auto v = state.allocValue(); @@ -199,7 +205,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - auto sourceInfo = fetchFlake(state, flakeRef); + FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); @@ -209,18 +215,16 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (state.allowedPaths) state.allowedPaths->insert(flakePath); - FlakeRef newFlakeRef(flakeRef); - if (std::get_if(&newFlakeRef.data)) { - FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef); - if (srcInfo.rev) { - std::string uri = flakeRef.baseRef().to_string(); - newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string(Base16, false)); - } + Flake flake(flakeRef); + if (std::get_if(&flakeRef.data)) { + if (sourceInfo.rev) + flake.ref = FlakeRef(flakeRef.baseRef().to_string() + + "/" + sourceInfo.rev->to_string(Base16, false)); } - Flake flake(newFlakeRef); flake.path = flakePath; flake.revCount = sourceInfo.revCount; + flake.path = flakePath; Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -242,6 +246,15 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) *(**requires).value->listElems()[n], *(**requires).pos))); } + if (std::optional nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) { + state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos); + for (Attr attr : *(*(**nonFlakeRequires).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef); + } + } + if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { state.forceFunction(*(**provides).value, *(**provides).pos); flake.vProvides = (**provides).value; @@ -250,86 +263,107 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack - if (pathExists(lockFile)) { - flake.lockFile = readRegistry(lockFile); - for (auto & entry : flake.lockFile->entries) - if (!entry.second.ref.isImmutable()) - throw Error("flake lock file '%s' contains mutable entry '%s'", - lockFile, entry.second.ref.to_string()); - } + flake.lockFile = readRegistry(lockFile); + for (auto & entry : flake.lockFile->entries) + if (!entry.second.ref.isImmutable()) + throw Error("flake lock file '%s' contains mutable entry '%s'", + lockFile, entry.second.ref.to_string()); + return flake; } +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId) +{ + FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); + debug("got non-flake source '%s' with revision %s", + sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + + auto flakePath = sourceInfo.storePath; + state.store->assertStorePath(flakePath); + + if (state.allowedPaths) + state.allowedPaths->insert(flakePath); + + NonFlake nonFlake(flakeRef); + if (std::get_if(&flakeRef.data)) { + if (sourceInfo.rev) + nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string() + + "/" + sourceInfo.rev->to_string(Base16, false)); + } + + nonFlake.path = flakePath; + + nonFlake.id = flakeId; + + return nonFlake; +} + /* Given a flake reference, recursively fetch it and its dependencies. FIXME: this should return a graph of flakes. */ -static std::tuple> resolveFlake(EvalState & state, - const FlakeRef & topRef, bool impureTopRef) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) { - std::map done; - std::queue> todo; - std::optional topFlakeId; /// FIXME: ambiguous - todo.push({topRef, true}); + Dependencies deps; + std::queue todo; + bool isTopLevel = true; + todo.push(topRef); auto registries = state.getFlakeRegistries(); - //std::shared_ptr localRegistry = registries.at(2); while (!todo.empty()) { - auto [flakeRef, toplevel] = todo.front(); + auto flakeRef = todo.front(); todo.pop(); if (auto refData = std::get_if(&flakeRef.data)) { if (done.count(refData->id)) continue; // optimization - flakeRef = lookupFlake(state, flakeRef, - !evalSettings.pureEval || (toplevel && impureTopRef) ? registries : std::vector>()); - // This is why we need the `registries`. - } - -#if 0 - if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) + flakeRef = lookupFlake(state, flakeRef, registries); + if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); -#endif auto flake = getFlake(state, flakeRef); - if (done.count(flake.id)) continue; + if (isTopLevel) { + deps.topFlakeId = flake.id; + isTopLevel = false; + } - if (toplevel) topFlakeId = flake.id; + for (auto & flakeRef : flake.requires) + todo.push(flakeRef); - for (auto & require : flake.requires) - todo.push({require, false}); + for (auto & x : flake.nonFlakeRequires) + deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first)); + // TODO (Nick): If there are 2 non-flake dependencies with the same + // FlakeId, this will lead to trouble! One of the dependencies won't + // be used! -#if 0 - // The following piece of code basically adds the FlakeRefs from - // the lockfiles of dependencies to the localRegistry. This is used - // to resolve future `FlakeId`s, in `lookupFlake` a bit above this. - if (flake.lockFile) - for (auto & entry : flake.lockFile->entries) { - if (localRegistry->entries.count(entry.first)) continue; - localRegistry->entries.emplace(entry.first, entry.second); - } -#endif - - done.emplace(flake.id, std::move(flake)); + deps.flakes.push_back(flake); } - assert(topFlakeId); - return {*topFlakeId, std::move(done)}; + return deps; } FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) { FlakeRegistry newLockFile; - std::map myDependencyMap = get<1>(resolveFlake(evalState, flakeRef, false)); + Dependencies deps = resolveFlake(evalState, flakeRef, false); // Nick assumed that "topRefPure" means that the Flake for flakeRef can be // fetched purely. - for (auto const& require : myDependencyMap) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.second.ref); - // The FlakeRefs are immutable because they come out of the Flake objects, - // not from the requires. - newLockFile.entries.insert(std::pair(require.first, entry)); + for (auto const& require : deps.flakes) { + FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref); + // The FlakeRefs are immutable because they come out of the Flake objects. + if (require.id != deps.topFlakeId) + newLockFile.entries.insert_or_assign(require.id, entry); + // TODO (Nick): If there are 2 flake dependencies with the same FlakeId, + // one of them gets ignored! + } + for (auto const& nonFlake : deps.nonFlakes) { + FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref); + newLockFile.entries.insert_or_assign(nonFlake.id, entry); + // We are assuming the sets of FlakeIds for flakes and non-flakes + // are disjoint. } return newLockFile; } @@ -348,21 +382,26 @@ void updateLockFile(EvalState & state, std::string path) } } -Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) +// Return the `provides` of the top flake, while assigning to `v` the provides +// of the dependencies as well. +Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, Value & v) { - auto [topFlakeId, flakes] = resolveFlake(state, flakeRef, impureTopRef); + FlakeRef flakeRef = FlakeRef(flakeUri); + + Dependencies deps = resolveFlake(state, flakeRef, impure); // FIXME: we should call each flake with only its dependencies // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); + // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - state.mkAttrs(*vResult, flakes.size()); + state.mkAttrs(*vResult, dependencies.flakes.size()); Value * vTop = 0; - for (auto & flake : flakes) { - auto vFlake = state.allocAttr(*vResult, flake.second.id); + for (auto & flake : deps.flakes) { + auto vFlake = state.allocAttr(*vResult, flake.id); if (topFlakeId == flake.second.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index aea4e8aa2..ffd962561 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -38,15 +38,34 @@ struct Flake std::optional revCount; std::vector requires; std::shared_ptr lockFile; + std::map nonFlakeRequires; Value * vProvides; // FIXME: gc - // commit hash // date // content hash - Flake(FlakeRef & flakeRef) : ref(flakeRef) {}; + Flake(const FlakeRef flakeRef) : ref(flakeRef) {}; +}; + +struct NonFlake +{ + FlakeId id; + FlakeRef ref; + Path path; + // date + // content hash + NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; Flake getFlake(EvalState &, const FlakeRef &); +struct Dependencies +{ + FlakeId topFlakeId; + std::vector flakes; + std::vector nonFlakes; +}; + +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); + FlakeRegistry updateLockFile(EvalState &, Flake &); void updateLockFile(EvalState &, std::string); diff --git a/src/nix/build.cc b/src/nix/build.cc index da7c7f614..a6fcf5094 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,7 +11,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; - std::optional gitRepo = std::nullopt; + bool updateLock = true; CmdBuild() { @@ -28,9 +28,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand .set(&outLink, Path("")); mkFlag() - .longName("update-lock-file") - .description("update the lock file") - .dest(&gitRepo); + .longName("no-update") + .description("don't update the lock file") + .set(&updateLock, false); } std::string name() override @@ -78,8 +78,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - if (gitRepo) - updateLockFile(*evalState, *gitRepo); + if(updateLock) + for (int i = 0; i < installables.size(); i++) + if (auto flakeUri = installableToFlakeUri) + updateLockFile(*evalState, FlakeRef(*flakeUri)); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3d2fb7832..07d31c45a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -36,6 +36,60 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; +void printFlakeInfo(Flake & flake, bool json) { + if (json) { + nlohmann::json j; + j["name"] = flake.id; + j["location"] = flake.path; + j["description"] = flake.description; + std::cout << j.dump(4) << std::endl; + } else { + std::cout << "Name: " << flake.id << "\n"; + std::cout << "Description: " << flake.description << "\n"; + std::cout << "Location: " << flake.path << "\n"; + } +} + +void printNonFlakeInfo(NonFlake & nonFlake, bool json) { + if (json) { + nlohmann::json j; + j["name"] = nonFlake.id; + j["location"] = nonFlake.path; + std::cout << j.dump(4) << std::endl; + } else { + std::cout << "name: " << nonFlake.id << "\n"; + std::cout << "Location: " << nonFlake.path << "\n"; + } +} + +struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs +{ + std::string name() override + { + return "deps"; + } + + std::string description() override + { + return "list informaton about dependencies"; + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + FlakeRef flakeRef(flakeUri); + + Dependencies deps = resolveFlake(*evalState, flakeRef, true); + + for (auto & flake : deps.flakes) + printFlakeInfo(flake, json); + + for (auto & nonFlake : deps.nonFlakes) + printNonFlakeInfo(nonFlake, json); + } +}; + struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs { std::string name() override @@ -73,15 +127,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand { auto evalState = std::make_shared(searchPath, store); nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); - if (json) { - nlohmann::json j; - j["location"] = flake.path; - j["description"] = flake.description; - std::cout << j.dump(4) << std::endl; - } else { - std::cout << "Description: " << flake.description << "\n"; - std::cout << "Location: " << flake.path << "\n"; - } + printFlakeInfo(flake, json); } }; @@ -218,6 +264,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command : MultiCommand({make_ref() , make_ref() , make_ref() + , make_ref() , make_ref() , make_ref() , make_ref() From 641db127be9df82fe4d51290120a8ba6d0b5f4fd Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 2/6] FlakeIds are now properly looked up in registries --- src/libexpr/primops/flake.cc | 23 +++++++++++++---------- src/libexpr/primops/flake.hh | 2 +- src/libexpr/primops/flakeref.hh | 18 ++++++++++++++++++ src/nix/build.cc | 7 ++++--- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c4ae29022..70d1b871a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -123,6 +123,10 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); + if (refData->ref) + newRef.setRef(*refData->ref); + if (refData->rev) + newRef.setRev(*refData->rev); return newRef; } } @@ -224,7 +228,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) flake.path = flakePath; flake.revCount = sourceInfo.revCount; - flake.path = flakePath; Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -317,9 +320,9 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impur auto flakeRef = todo.front(); todo.pop(); - if (auto refData = std::get_if(&flakeRef.data)) { - if (done.count(refData->id)) continue; // optimization + if (std::get_if(&flakeRef.data)) flakeRef = lookupFlake(state, flakeRef, registries); + if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); @@ -368,7 +371,7 @@ FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) return newLockFile; } -void updateLockFile(EvalState & state, std::string path) +void updateLockFile(EvalState & state, Path path) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); @@ -384,7 +387,7 @@ void updateLockFile(EvalState & state, std::string path) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, Value & v) +Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) { FlakeRef flakeRef = FlakeRef(flakeUri); @@ -406,16 +409,16 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, state.mkAttrs(*vFlake, 4); - mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); + mkString(*state.allocAttr(*vFlake, state.sDescription), flake.description); - state.store->assertStorePath(flake.second.path); - mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.second.path, {flake.second.path}); + state.store->assertStorePath(flake.path); + mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path}); if (flake.second.revCount) - mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.second.revCount); + mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); - mkApp(*vProvides, *flake.second.vProvides, *vResult); + mkApp(*vProvides, *flake.vProvides, *vResult); // Should this be vResult or vFlake??? Or both! vFlake->attrs->sort(); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index ffd962561..a8f907784 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -68,5 +68,5 @@ Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); FlakeRegistry updateLockFile(EvalState &, Flake &); -void updateLockFile(EvalState &, std::string); +void updateLockFile(EvalState &, Path); } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index fa14f7c25..94a75fb2b 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -160,5 +160,23 @@ struct FlakeRef bool isImmutable() const; FlakeRef baseRef() const; + + void setRef(std::optional ref) { + if (auto refData = std::get_if(&data)) + refData->ref = ref; + else if (auto refData = std::get_if(&data)) + refData->ref = ref; + else if (auto refData = std::get_if(&data)) + refData->ref = ref; + } + + void setRev(std::optional rev) { + if (auto refData = std::get_if(&data)) + refData->rev = rev; + else if (auto refData = std::get_if(&data)) + refData->rev = rev; + else if (auto refData = std::get_if(&data)) + refData->rev = rev; + } }; } diff --git a/src/nix/build.cc b/src/nix/build.cc index a6fcf5094..f6908b0c0 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,10 +78,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } + std::string flakeUri = ""; if(updateLock) - for (int i = 0; i < installables.size(); i++) - if (auto flakeUri = installableToFlakeUri) - updateLockFile(*evalState, FlakeRef(*flakeUri)); + for (uint i = 0; i < installables.size(); i++) + // if (auto flakeUri = installableToFlakeUri) + updateLockFile(*evalState, flakeUri); } }; From 3ec0c82fab94533807c5c3bb25df2b43d8339ed3 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Fri, 29 Mar 2019 16:18:25 +0100 Subject: [PATCH 3/6] Fixed dependency resolution --- src/libexpr/primops/flake.cc | 204 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 31 ++++-- src/nix/build.cc | 10 +- src/nix/flake.cc | 25 +++-- 4 files changed, 168 insertions(+), 102 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 70d1b871a..1ad9ad3f8 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -39,9 +39,8 @@ std::shared_ptr readRegistry(const Path & path) /* Write the registry or lock file to a file. */ void writeRegistry(FlakeRegistry registry, Path path) { - nlohmann::json json = {}; + nlohmann::json json; json["version"] = 1; - json["flakes"] = {}; for (auto elem : registry.entries) { json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; } @@ -49,6 +48,85 @@ void writeRegistry(FlakeRegistry registry, Path path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } +LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + + LockFile::FlakeEntry entry(flakeRef); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + for (auto & x : entry.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : entry.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + return json; +} + +void writeLockFile(LockFile lockFile, Path path) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeRequires"]; + for (auto & x : lockFile.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // '4' = indentation in json file +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -142,17 +220,13 @@ struct FlakeSourceInfo std::optional revCount; }; -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeRef directFlakeRef = FlakeRef(flakeRef); - if (!flakeRef.isDirect()) { - directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); - } - assert(directFlakeRef.isDirect()); - // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes.. + FlakeRef directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); if (auto refData = std::get_if(&directFlakeRef.data)) { - // FIXME: require hash in pure mode. + if (evalSettings.pureEval && !impureIsAllowed && !directFlakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", directFlakeRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -207,7 +281,8 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } -Flake getFlake(EvalState & state, const FlakeRef & flakeRef) +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", @@ -264,14 +339,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake lacks attribute 'provides'"); - auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack - - flake.lockFile = readRegistry(lockFile); - for (auto & entry : flake.lockFile->entries) - if (!entry.second.ref.isImmutable()) - throw Error("flake lock file '%s' contains mutable entry '%s'", - lockFile, entry.second.ref.to_string()); + const Path lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack + flake.lockFile = readLockFile(lockFile); return flake; } @@ -307,77 +377,50 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flake dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake = true) { - Dependencies deps; - std::queue todo; - bool isTopLevel = true; - todo.push(topRef); + Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); + Dependencies deps(flake); - auto registries = state.getFlakeRegistries(); + for (auto & nonFlakeInfo : flake.nonFlakeRequires) + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); - while (!todo.empty()) { - auto flakeRef = todo.front(); - todo.pop(); - - if (std::get_if(&flakeRef.data)) - flakeRef = lookupFlake(state, flakeRef, registries); - - if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) - throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); - - auto flake = getFlake(state, flakeRef); - - if (isTopLevel) { - deps.topFlakeId = flake.id; - isTopLevel = false; - } - - for (auto & flakeRef : flake.requires) - todo.push(flakeRef); - - for (auto & x : flake.nonFlakeRequires) - deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first)); - // TODO (Nick): If there are 2 non-flake dependencies with the same - // FlakeId, this will lead to trouble! One of the dependencies won't - // be used! - - deps.flakes.push_back(flake); - } + for (auto & newFlakeRef : flake.requires) + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, impureTopRef, false)); return deps; } -FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) +LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) { - FlakeRegistry newLockFile; - Dependencies deps = resolveFlake(evalState, flakeRef, false); - // Nick assumed that "topRefPure" means that the Flake for flakeRef can be - // fetched purely. - for (auto const& require : deps.flakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref); - // The FlakeRefs are immutable because they come out of the Flake objects. - if (require.id != deps.topFlakeId) - newLockFile.entries.insert_or_assign(require.id, entry); - // TODO (Nick): If there are 2 flake dependencies with the same FlakeId, - // one of them gets ignored! - } - for (auto const& nonFlake : deps.nonFlakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref); - newLockFile.entries.insert_or_assign(nonFlake.id, entry); - // We are assuming the sets of FlakeIds for flakes and non-flakes - // are disjoint. - } - return newLockFile; + LockFile::FlakeEntry entry(deps.flake.ref); + + for (Dependencies & deps : deps.flakeDeps) + entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); + + for (NonFlake & nonFlake : deps.nonFlakeDeps) + entry.nonFlakeEntries.insert_or_assign(nonFlake.id, nonFlake.ref); + + return entry; } -void updateLockFile(EvalState & state, Path path) +LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopRef) +{ + Dependencies deps = resolveFlake(evalState, flakeRef, impureTopRef); + LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); + LockFile lockFile; + lockFile.flakeEntries = entry.flakeEntries; + lockFile.nonFlakeEntries = entry.nonFlakeEntries; + return lockFile; +} + +void updateLockFile(EvalState & state, Path path, bool impureTopRef) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { - FlakeRegistry newLockFile = updateLockFile(state, flakeRef); - writeRegistry(newLockFile, path + "/flake.lock"); + LockFile lockFile = getLockFile(state, flakeRef, impureTopRef); + writeLockFile(lockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { @@ -393,19 +436,19 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) Dependencies deps = resolveFlake(state, flakeRef, impure); - // FIXME: we should call each flake with only its dependencies - // (rather than the closure of the top-level flake). + // // FIXME: we should call each flake with only its dependencies + // // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - state.mkAttrs(*vResult, dependencies.flakes.size()); + state.mkAttrs(*vResult, deps.flakeDeps.size()); Value * vTop = 0; - for (auto & flake : deps.flakes) { + for (auto & flake : deps.flakeDeps) { auto vFlake = state.allocAttr(*vResult, flake.id); - if (topFlakeId == flake.second.id) vTop = vFlake; + if (deps.topFlakeId == flake.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); @@ -431,6 +474,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) return vTop; } +// This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index a8f907784..019688f37 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -19,6 +19,20 @@ struct FlakeRegistry std::map entries; }; +struct LockFile +{ + struct FlakeEntry + { + FlakeRef ref; + std::map flakeEntries; + std::map nonFlakeEntries; + FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; + }; + + std::map flakeEntries; + std::map nonFlakeEntries; +}; + Path getUserRegistryPath(); Value * makeFlakeRegistryValue(EvalState & state); @@ -37,7 +51,7 @@ struct Flake Path path; std::optional revCount; std::vector requires; - std::shared_ptr lockFile; + LockFile lockFile; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc // date @@ -55,18 +69,17 @@ struct NonFlake NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; -Flake getFlake(EvalState &, const FlakeRef &); +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct Dependencies { - FlakeId topFlakeId; - std::vector flakes; - std::vector nonFlakes; + Flake flake; + std::vector flakeDeps; // The flake dependencies + std::vector nonFlakeDeps; + Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); -FlakeRegistry updateLockFile(EvalState &, Flake &); - -void updateLockFile(EvalState &, Path); +void updateLockFile(EvalState &, Path path, bool impureTopRef); } diff --git a/src/nix/build.cc b/src/nix/build.cc index f6908b0c0..226c21e9e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,11 +78,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - std::string flakeUri = ""; - if(updateLock) - for (uint i = 0; i < installables.size(); i++) - // if (auto flakeUri = installableToFlakeUri) - updateLockFile(*evalState, flakeUri); + // std::string flakeUri = ""; + // if(updateLock) + // for (uint i = 0; i < installables.size(); i++) + // // if (auto flakeUri = installableToFlakeUri) + // updateLockFile(*evalState, flakeUri, true); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 07d31c45a..ff291aa80 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -5,6 +5,7 @@ #include "progress-bar.hh" #include "eval.hh" #include +#include using namespace nix; @@ -80,13 +81,21 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, true, true); - for (auto & flake : deps.flakes) - printFlakeInfo(flake, json); + std::queue todo; + todo.push(deps); - for (auto & nonFlake : deps.nonFlakes) - printNonFlakeInfo(nonFlake, json); + while (!todo.empty()) { + deps = todo.front(); + todo.pop(); + + for (auto & nonFlake : deps.nonFlakeDeps) + printNonFlakeInfo(nonFlake, json); + + for (auto & newDeps : deps.flakeDeps) + todo.push(newDeps); + } } }; @@ -107,7 +116,7 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs auto evalState = std::make_shared(searchPath, store); if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath); + updateLockFile(*evalState, gitPath, true); } }; @@ -126,7 +135,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; @@ -220,7 +229,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs auto it = userRegistry.entries.find(flakeId); if (it != userRegistry.entries.end()) { FlakeRef oldRef = it->second.ref; - it->second.ref = getFlake(*evalState, oldRef).ref; + it->second.ref = getFlake(*evalState, oldRef, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else From f39670c6318ba8d2260b3ac54f46161d74649266 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Sat, 6 Apr 2019 20:45:35 +0200 Subject: [PATCH 4/6] Took ref and rev out of FlakeRef --- src/libexpr/primops/flake.cc | 31 +++++++-------- src/libexpr/primops/flakeref.cc | 69 +++++++++++---------------------- src/libexpr/primops/flakeref.hh | 28 +++---------- 3 files changed, 41 insertions(+), 87 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1ad9ad3f8..cea3854e4 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -201,10 +201,8 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); - if (refData->ref) - newRef.setRef(*refData->ref); - if (refData->rev) - newRef.setRev(*refData->rev); + if (flakeRef.ref) newRef.setRef(*flakeRef.ref); + if (flakeRef.rev) newRef.setRev(*flakeRef.rev); return newRef; } } @@ -220,13 +218,13 @@ struct FlakeSourceInfo std::optional revCount; }; -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) { - FlakeRef directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); + FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); - if (auto refData = std::get_if(&directFlakeRef.data)) { - if (evalSettings.pureEval && !impureIsAllowed && !directFlakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", directFlakeRef.to_string()); + if (auto refData = std::get_if(&fRef.data)) { + if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -235,14 +233,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo, - refData->rev - ? refData->rev->to_string(Base16, false) - : refData->ref - ? *refData->ref - : "master"); + fRef.rev ? fRef.rev->to_string(Base16, false) + : fRef.ref ? *fRef.ref : "master"); auto result = getDownloader()->downloadCached(state.store, url, true, "source", - Hash(), nullptr, refData->rev ? 1000000000 : settings.tarballTtl); + Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); @@ -257,9 +252,9 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, return info; } - else if (auto refData = std::get_if(&directFlakeRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, refData->ref, - refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); + else if (auto refData = std::get_if(&fRef.data)) { + auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, + fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); FlakeSourceInfo info; info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 1df53bfb8..f160b257b 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -58,11 +58,11 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) IsFlakeId d; d.id = match[1]; if (match[2].matched) - d.rev = Hash(match[2], htSHA1); + rev = Hash(match[2], htSHA1); else if (match[3].matched) { - d.ref = match[3]; + ref = match[3]; if (match[4].matched) - d.rev = Hash(match[4], htSHA1); + rev = Hash(match[4], htSHA1); } data = d; } @@ -72,9 +72,9 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) d.owner = match[1]; d.repo = match[2]; if (match[3].matched) - d.rev = Hash(match[3], htSHA1); + rev = Hash(match[3], htSHA1); else if (match[4].matched) { - d.ref = match[4]; + ref = match[4]; } data = d; } @@ -92,16 +92,16 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) if (name == "rev") { if (!std::regex_match(value, revRegex)) throw Error("invalid Git revision '%s'", value); - d.rev = Hash(value, htSHA1); + rev = Hash(value, htSHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex2)) throw Error("invalid Git ref '%s'", value); - d.ref = value; + ref = value; } else // FIXME: should probably pass through unknown parameters throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); } - if (d.rev && !d.ref) + if (rev && !ref) throw Error("flake URI '%s' lacks a Git ref", uri); data = d; } @@ -118,27 +118,18 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { - if (auto refData = std::get_if(&data)) { - return - "flake:" + refData->id + - (refData->ref ? "/" + *refData->ref : "") + - (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); - } + std::string string; + if (auto refData = std::get_if(&data)) + string = "flake:" + refData->id; else if (auto refData = std::get_if(&data)) { - assert(!refData->ref || !refData->rev); - return - "github:" + refData->owner + "/" + refData->repo + - (refData->ref ? "/" + *refData->ref : "") + - (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); + assert(!ref || !rev); + string = "github:" + refData->owner + "/" + refData->repo; } else if (auto refData = std::get_if(&data)) { - assert(refData->ref || !refData->rev); - return - refData->uri + - (refData->ref ? "?ref=" + *refData->ref : "") + - (refData->rev ? "&rev=" + refData->rev->to_string(Base16, false) : ""); + assert(ref || !rev); + string = refData->uri; } else if (auto refData = std::get_if(&data)) { @@ -146,38 +137,22 @@ std::string FlakeRef::to_string() const } else abort(); + + string += (ref ? "/" + *ref : "") + + (rev ? "/" + rev->to_string(Base16, false) : ""); + return string; } bool FlakeRef::isImmutable() const { - if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (std::get_if(&data)) - return false; - - else abort(); + return (bool) rev; } FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. { FlakeRef result(*this); - if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } else if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } else if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } + result.ref = std::nullopt; + result.rev = std::nullopt; return result; } } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 94a75fb2b..9276fc737 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -101,25 +101,23 @@ typedef std::string FlakeId; struct FlakeRef { + std::optional ref; + std::optional rev; + struct IsFlakeId { FlakeId id; - std::optional ref; - std::optional rev; }; struct IsGitHub { std::string owner, repo; - std::optional ref; - std::optional rev; }; + // Git, Tarball struct IsGit { std::string uri; - std::optional ref; - std::optional rev; }; struct IsPath @@ -161,22 +159,8 @@ struct FlakeRef FlakeRef baseRef() const; - void setRef(std::optional ref) { - if (auto refData = std::get_if(&data)) - refData->ref = ref; - else if (auto refData = std::get_if(&data)) - refData->ref = ref; - else if (auto refData = std::get_if(&data)) - refData->ref = ref; - } + void setRef(std::optional ref) { ref = ref; } - void setRev(std::optional rev) { - if (auto refData = std::get_if(&data)) - refData->rev = rev; - else if (auto refData = std::get_if(&data)) - refData->rev = rev; - else if (auto refData = std::get_if(&data)) - refData->rev = rev; - } + void setRev(std::optional rev) { rev = rev; } }; } From c64f98b883515df70e2457ae01070b5af9ae69b9 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 5/6] FlakeAlias is implemented --- src/libexpr/primops/flake.cc | 104 ++++++++++++++++++++++++++++---- src/libexpr/primops/flake.hh | 10 +-- src/libexpr/primops/flakeref.cc | 15 +++-- src/libexpr/primops/flakeref.hh | 8 ++- src/nix/build.cc | 2 +- src/nix/flake.cc | 36 +++++------ 6 files changed, 129 insertions(+), 46 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index cea3854e4..729b1da95 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -127,6 +127,86 @@ void writeLockFile(LockFile lockFile, Path path) writeFile(path, json.dump(4)); // '4' = indentation in json file } +Path getUserRegistryPath() +>>>>>>> Fixed dependency resolution +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + + LockFile::FlakeEntry entry(flakeRef); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + for (auto & x : entry.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : entry.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + return json; +} + +void writeLockFile(LockFile lockFile, Path path) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeRequires"]; + for (auto & x : lockFile.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // '4' = indentation in json file +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -194,9 +274,9 @@ Value * makeFlakeRegistryValue(EvalState & state) static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, std::vector> registries) { - if (auto refData = std::get_if(&flakeRef.data)) { + if (auto refData = std::get_if(&flakeRef.data)) { for (auto registry : registries) { - auto i = registry->entries.find(refData->id); + auto i = registry->entries.find(refData->alias); if (i != registry->entries.end()) { auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) @@ -206,7 +286,7 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return newRef; } } - throw Error("cannot find flake '%s' in the flake registry or in the flake lock file", refData->id); + throw Error("cannot find flake with alias '%s' in the flake registry or in the flake lock file", refData->alias); } else return flakeRef; } @@ -342,7 +422,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } // Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias) { FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got non-flake source '%s' with revision %s", @@ -363,7 +443,7 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flake nonFlake.path = flakePath; - nonFlake.id = flakeId; + nonFlake.alias = alias; return nonFlake; } @@ -394,14 +474,14 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); for (NonFlake & nonFlake : deps.nonFlakeDeps) - entry.nonFlakeEntries.insert_or_assign(nonFlake.id, nonFlake.ref); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); return entry; } -LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopRef) +LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef) { - Dependencies deps = resolveFlake(evalState, flakeRef, impureTopRef); + Dependencies deps = resolveFlake(evalState, flakeRef, true); LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); LockFile lockFile; lockFile.flakeEntries = entry.flakeEntries; @@ -409,17 +489,17 @@ LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopR return lockFile; } -void updateLockFile(EvalState & state, Path path, bool impureTopRef) +void updateLockFile(EvalState & state, Path path) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { - LockFile lockFile = getLockFile(state, flakeRef, impureTopRef); + LockFile lockFile = getLockFile(state, flakeRef); writeLockFile(lockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { - throw UsageError("you can only update local flakes, not flakes through their FlakeId"); + throw UsageError("you can only update local flakes, not flakes through their FlakeAlias"); } } @@ -456,7 +536,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); - mkApp(*vProvides, *flake.vProvides, *vResult); // Should this be vResult or vFlake??? Or both! + mkApp(*vProvides, *flake.vProvides, *vResult); vFlake->attrs->sort(); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 019688f37..adf8b07af 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -16,7 +16,7 @@ struct FlakeRegistry Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; Entry operator=(const Entry & entry) { return Entry(entry.ref); } }; - std::map entries; + std::map entries; }; struct LockFile @@ -52,7 +52,7 @@ struct Flake std::optional revCount; std::vector requires; LockFile lockFile; - std::map nonFlakeRequires; + std::map nonFlakeRequires; Value * vProvides; // FIXME: gc // date // content hash @@ -61,7 +61,7 @@ struct Flake struct NonFlake { - FlakeId id; + FlakeAlias alias; FlakeRef ref; Path path; // date @@ -81,5 +81,7 @@ struct Dependencies Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); -void updateLockFile(EvalState &, Path path, bool impureTopRef); +FlakeRegistry updateLockFile(EvalState &, Flake &); + +void updateLockFile(EvalState &, Path path); } diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index f160b257b..ab1e5e152 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -19,7 +19,7 @@ const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + // "master/e72daba8250068216d79d2aeef40d4d95aff6666"). const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; // GitHub references. const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; @@ -37,7 +37,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) // FIXME: could combine this into one regex. static std::regex flakeRegex( - "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?", + "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", std::regex::ECMAScript); static std::regex githubRegex( @@ -55,8 +55,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::cmatch match; if (std::regex_match(uri.c_str(), match, flakeRegex)) { - IsFlakeId d; - d.id = match[1]; + IsAlias d; + d.alias = match[1]; if (match[2].matched) rev = Hash(match[2], htSHA1); else if (match[3].matched) { @@ -119,8 +119,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { std::string string; - if (auto refData = std::get_if(&data)) - string = "flake:" + refData->id; + if (auto refData = std::get_if(&data)) + string = "flake:" + refData->alias; else if (auto refData = std::get_if(&data)) { assert(!ref || !rev); @@ -132,9 +132,8 @@ std::string FlakeRef::to_string() const string = refData->uri; } - else if (auto refData = std::get_if(&data)) { + else if (auto refData = std::get_if(&data)) return refData->path; - } else abort(); diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 9276fc737..32904953a 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -98,15 +98,17 @@ namespace nix { */ typedef std::string FlakeId; +typedef std::string FlakeAlias; +typedef std::string FlakeUri; struct FlakeRef { std::optional ref; std::optional rev; - struct IsFlakeId + struct IsAlias { - FlakeId id; + FlakeAlias alias; }; struct IsGitHub @@ -150,7 +152,7 @@ struct FlakeRef a flake ID, which requires a lookup in the flake registry. */ bool isDirect() const { - return !std::get_if(&data); + return !std::get_if(&data); } /* Check whether this is an "immutable" flake reference, that is, diff --git a/src/nix/build.cc b/src/nix/build.cc index 226c21e9e..a2fc56e69 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -82,7 +82,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand // if(updateLock) // for (uint i = 0; i < installables.size(); i++) // // if (auto flakeUri = installableToFlakeUri) - // updateLockFile(*evalState, flakeUri, true); + // updateLockFile(*evalState, flakeUri); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ff291aa80..df944a148 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -40,12 +40,12 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs void printFlakeInfo(Flake & flake, bool json) { if (json) { nlohmann::json j; - j["name"] = flake.id; + j["id"] = flake.id; j["location"] = flake.path; j["description"] = flake.description; std::cout << j.dump(4) << std::endl; } else { - std::cout << "Name: " << flake.id << "\n"; + std::cout << "ID: " << flake.id << "\n"; std::cout << "Description: " << flake.description << "\n"; std::cout << "Location: " << flake.path << "\n"; } @@ -54,11 +54,11 @@ void printFlakeInfo(Flake & flake, bool json) { void printNonFlakeInfo(NonFlake & nonFlake, bool json) { if (json) { nlohmann::json j; - j["name"] = nonFlake.id; + j["name"] = nonFlake.alias; j["location"] = nonFlake.path; std::cout << j.dump(4) << std::endl; } else { - std::cout << "name: " << nonFlake.id << "\n"; + std::cout << "name: " << nonFlake.alias << "\n"; std::cout << "Location: " << nonFlake.path << "\n"; } } @@ -116,7 +116,7 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs auto evalState = std::make_shared(searchPath, store); if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath, true); + updateLockFile(*evalState, gitPath); } }; @@ -135,15 +135,15 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); printFlakeInfo(flake, json); } }; struct CmdFlakeAdd : MixEvalArgs, Command { - std::string flakeId; - std::string flakeUri; + FlakeAlias flakeAlias; + FlakeUri flakeUri; std::string name() override { @@ -157,7 +157,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command CmdFlakeAdd() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); expectArg("flake-uri", &flakeUri); } @@ -167,15 +167,15 @@ struct CmdFlakeAdd : MixEvalArgs, Command Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); FlakeRegistry::Entry entry(newFlakeRef); - userRegistry->entries.erase(flakeId); - userRegistry->entries.insert_or_assign(flakeId, newFlakeRef); + userRegistry->entries.erase(flakeAlias); + userRegistry->entries.insert_or_assign(flakeAlias, newFlakeRef); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command { - std::string flakeId; + FlakeAlias flakeAlias; std::string name() override { @@ -189,21 +189,21 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); } void run() override { Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - userRegistry->entries.erase(flakeId); + userRegistry->entries.erase(flakeAlias); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs { - std::string flakeId; + FlakeAlias flakeAlias; std::string name() override { @@ -217,7 +217,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); } void run(nix::ref store) override @@ -226,14 +226,14 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs Path userRegistryPath = getUserRegistryPath(); FlakeRegistry userRegistry = *readRegistry(userRegistryPath); - auto it = userRegistry.entries.find(flakeId); + auto it = userRegistry.entries.find(flakeAlias); if (it != userRegistry.entries.end()) { FlakeRef oldRef = it->second.ref; it->second.ref = getFlake(*evalState, oldRef, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else - throw Error("the flake identifier '%s' does not exist in the user registry", flakeId); + throw Error("the flake alias '%s' does not exist in the user registry", flakeAlias); } }; From 4ad4e4866891a62a6e1bb919d81e224ba0a1cf1c Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Mon, 8 Apr 2019 19:03:00 +0200 Subject: [PATCH 6/6] FlakeRegistry = FlakeRef -> FlakeRef --- flake-registry.json | 4 +- src/libexpr/primops/flake.cc | 169 +++++++++----------------------- src/libexpr/primops/flake.hh | 10 +- src/libexpr/primops/flakeref.hh | 50 ++++++---- src/nix/build.cc | 3 +- src/nix/command.hh | 3 +- src/nix/flake.cc | 45 ++++----- src/nix/installables.cc | 1 - 8 files changed, 105 insertions(+), 180 deletions(-) diff --git a/flake-registry.json b/flake-registry.json index b850daa74..378290ec6 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -1,5 +1,4 @@ { - "version": 1, "flakes": { "dwarffs": { "uri": "github:edolstra/dwarffs/flake" @@ -7,5 +6,6 @@ "nixpkgs": { "uri": "github:edolstra/nixpkgs/flake" } - } + }, + "version": 1 } diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 729b1da95..145d79446 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -28,10 +28,8 @@ std::shared_ptr readRegistry(const Path & path) throw Error("flake registry '%s' has unsupported version %d", path, version); auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - registry->entries.emplace(i.key(), entry); - } + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); return registry; } @@ -41,9 +39,8 @@ void writeRegistry(FlakeRegistry registry, Path path) { nlohmann::json json; json["version"] = 1; - for (auto elem : registry.entries) { - json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; - } + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } @@ -127,106 +124,31 @@ void writeLockFile(LockFile lockFile, Path path) writeFile(path, json.dump(4)); // '4' = indentation in json file } -Path getUserRegistryPath() ->>>>>>> Fixed dependency resolution +std::shared_ptr getGlobalRegistry() { - FlakeRef flakeRef(json["uri"]); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - - LockFile::FlakeEntry entry(flakeRef); - - auto nonFlakeRequires = json["nonFlakeRequires"]; - - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); - } - - auto requires = json["requires"]; - - for (auto i = requires.begin(); i != requires.end(); ++i) - entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return entry; -} - -LockFile readLockFile(const Path & path) -{ - LockFile lockFile; - - if (!pathExists(path)) - return lockFile; - - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("lock file '%s' has unsupported version %d", path, version); - - auto nonFlakeRequires = json["nonFlakeRequires"]; - - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); - } - - auto requires = json["requires"]; - - for (auto i = requires.begin(); i != requires.end(); ++i) - lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return lockFile; -} - -nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) -{ - nlohmann::json json; - json["uri"] = entry.ref.to_string(); - for (auto & x : entry.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); - for (auto & x : entry.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); - return json; -} - -void writeLockFile(LockFile lockFile, Path path) -{ - nlohmann::json json; - json["version"] = 1; - json["nonFlakeRequires"]; - for (auto & x : lockFile.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); - for (auto & x : lockFile.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); - createDirs(dirOf(path)); - writeFile(path, json.dump(4)); // '4' = indentation in json file + return std::make_shared(); } Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; } -std::shared_ptr getGlobalRegistry() -{ - // FIXME: get from nixos.org. - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); -} std::shared_ptr getUserRegistry() { return readRegistry(getUserRegistryPath()); } +std::shared_ptr getLocalRegistry() +{ + Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; + return readRegistry(registryFile); +} + std::shared_ptr getFlagRegistry() { + // TODO (Nick): Implement this. return std::make_shared(); - // TODO: Implement this once the right flags are implemented. } const std::vector> EvalState::getFlakeRegistries() @@ -259,9 +181,9 @@ Value * makeFlakeRegistryValue(EvalState & state) for (auto & registry : registries) { for (auto & entry : registry->entries) { - auto vEntry = state.allocAttr(*v, entry.first); + auto vEntry = state.allocAttr(*v, entry.first.to_string()); state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.to_string()); vEntry->attrs->sort(); } } @@ -272,23 +194,30 @@ Value * makeFlakeRegistryValue(EvalState & state) } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, - std::vector> registries) + std::vector> registries, std::vector pastSearches = {}) { - if (auto refData = std::get_if(&flakeRef.data)) { - for (auto registry : registries) { - auto i = registry->entries.find(refData->alias); - if (i != registry->entries.end()) { - auto newRef = FlakeRef(i->second.ref); - if (!newRef.isDirect()) - throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); - if (flakeRef.ref) newRef.setRef(*flakeRef.ref); - if (flakeRef.rev) newRef.setRev(*flakeRef.rev); - return newRef; + for (std::shared_ptr registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + if (std::get_if(&flakeRef.data)) { + if (flakeRef.ref) newRef.ref = flakeRef.ref; + if (flakeRef.rev) newRef.rev = flakeRef.rev; } + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); } - throw Error("cannot find flake with alias '%s' in the flake registry or in the flake lock file", refData->alias); - } else - return flakeRef; + } + if (!flakeRef.isDirect()) + throw Error("indirect flake URI '%s' is the result of a lookup", flakeRef.to_string()); + return flakeRef; } struct FlakeSourceInfo @@ -302,6 +231,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo { FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); + // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&fRef.data)) { if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string()); @@ -332,6 +262,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo return info; } + // This downloads the entire git history else if (auto refData = std::get_if(&fRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); @@ -342,7 +273,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo return info; } - else if (auto refData = std::get_if(&directFlakeRef.data)) { + else if (auto refData = std::get_if(&fRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, "", "source"); @@ -452,7 +383,7 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake = true) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake) { Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); Dependencies deps(flake); @@ -461,7 +392,7 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impur deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); for (auto & newFlakeRef : flake.requires) - deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, impureTopRef, false)); + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, false)); return deps; } @@ -505,25 +436,23 @@ void updateLockFile(EvalState & state, Path path) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) +Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) { - FlakeRef flakeRef = FlakeRef(flakeUri); + Dependencies deps = resolveFlake(state, flakeRef, impureTopRef); - Dependencies deps = resolveFlake(state, flakeRef, impure); - - // // FIXME: we should call each flake with only its dependencies - // // (rather than the closure of the top-level flake). + // FIXME: we should call each flake with only its dependencies + // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. state.mkAttrs(*vResult, deps.flakeDeps.size()); - Value * vTop = 0; + Value * vTop = state.allocAttr(*vResult, deps.flake.id); - for (auto & flake : deps.flakeDeps) { + for (auto & dep : deps.flakeDeps) { + Flake flake = dep.flake; auto vFlake = state.allocAttr(*vResult, flake.id); - if (deps.topFlakeId == flake.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); @@ -532,7 +461,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) state.store->assertStorePath(flake.path); mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path}); - if (flake.second.revCount) + if (flake.revCount) mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index adf8b07af..9da065234 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -10,13 +10,7 @@ class EvalState; struct FlakeRegistry { - struct Entry - { - FlakeRef ref; - Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; - Entry operator=(const Entry & entry) { return Entry(entry.ref); } - }; - std::map entries; + std::map entries; }; struct LockFile @@ -79,7 +73,7 @@ struct Dependencies Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake = true); FlakeRegistry updateLockFile(EvalState &, Flake &); diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 32904953a..d789a6f70 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -103,48 +103,60 @@ typedef std::string FlakeUri; struct FlakeRef { - std::optional ref; - std::optional rev; - struct IsAlias { FlakeAlias alias; + bool operator<(const IsAlias & b) const { return alias < b.alias; }; + bool operator==(const IsAlias & b) const { return alias == b.alias; }; }; - struct IsGitHub - { + struct IsGitHub { std::string owner, repo; + bool operator<(const IsGitHub & b) const { + return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); + } + bool operator==(const IsGitHub & b) const { + return owner == b.owner && repo == b.repo; + } }; // Git, Tarball struct IsGit { std::string uri; + bool operator<(const IsGit & b) const { return uri < b.uri; } + bool operator==(const IsGit & b) const { return uri == b.uri; } }; struct IsPath { Path path; + bool operator<(const IsPath & b) const { return path < b.path; } + bool operator==(const IsPath & b) const { return path == b.path; } }; // Git, Tarball - std::variant data; + std::variant data; + + std::optional ref; + std::optional rev; + + bool operator<(const FlakeRef & flakeRef) const + { + return std::make_tuple(this->data, ref, rev) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + } + + bool operator==(const FlakeRef & flakeRef) const + { + return std::make_tuple(this->data, ref, rev) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + } // Parse a flake URI. FlakeRef(const std::string & uri, bool allowRelative = false); - // Default constructor - FlakeRef(const FlakeRef & flakeRef) : data(flakeRef.data) {}; - - /* Unify two flake references so that the resulting reference - combines the information from both. For example, - "nixpkgs/" and "github:NixOS/nixpkgs" unifies to - "nixpkgs/master". May throw an exception if the references are - incompatible (e.g. "nixpkgs/" and "nixpkgs/", - where hash1 != hash2). */ - FlakeRef(const FlakeRef & a, const FlakeRef & b); - // FIXME: change to operator <<. std::string to_string() const; @@ -160,9 +172,5 @@ struct FlakeRef bool isImmutable() const; FlakeRef baseRef() const; - - void setRef(std::optional ref) { ref = ref; } - - void setRev(std::optional rev) { rev = rev; } }; } diff --git a/src/nix/build.cc b/src/nix/build.cc index a2fc56e69..5a3d9d31a 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,4 +1,3 @@ -#include "primops/flake.hh" #include "eval.hh" #include "command.hh" #include "common-args.hh" @@ -78,7 +77,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - // std::string flakeUri = ""; + // FlakeUri flakeUri = ""; // if(updateLock) // for (uint i = 0; i < installables.size(); i++) // // if (auto flakeUri = installableToFlakeUri) diff --git a/src/nix/command.hh b/src/nix/command.hh index 83959bf9a..56e1e6f34 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,6 +1,7 @@ #pragma once #include "args.hh" +#include "primops/flake.hh" #include "common-eval-args.hh" namespace nix { @@ -46,7 +47,7 @@ struct GitRepoCommand : virtual Args struct FlakeCommand : virtual Args { - std::string flakeUri; + FlakeUri flakeUri; FlakeCommand() { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index df944a148..dbf0d3e9a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,4 +1,3 @@ -#include "primops/flake.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -29,11 +28,9 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs stopProgressBar(); - for (auto & registry : registries) { - for (auto & entry : registry->entries) { - std::cout << entry.first << " " << entry.second.ref.to_string() << "\n"; - } - } + for (auto & registry : registries) + for (auto & entry : registry->entries) + std::cout << entry.first.to_string() << " " << entry.second.to_string() << "\n"; } }; @@ -81,7 +78,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, true); std::queue todo; todo.push(deps); @@ -135,15 +132,15 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; struct CmdFlakeAdd : MixEvalArgs, Command { - FlakeAlias flakeAlias; - FlakeUri flakeUri; + FlakeUri alias; + FlakeUri uri; std::string name() override { @@ -157,25 +154,24 @@ struct CmdFlakeAdd : MixEvalArgs, Command CmdFlakeAdd() { - expectArg("flake-id", &flakeAlias); - expectArg("flake-uri", &flakeUri); + expectArg("alias", &alias); + expectArg("flake-uri", &uri); } void run() override { - FlakeRef newFlakeRef(flakeUri); + FlakeRef aliasRef(alias); Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - FlakeRegistry::Entry entry(newFlakeRef); - userRegistry->entries.erase(flakeAlias); - userRegistry->entries.insert_or_assign(flakeAlias, newFlakeRef); + userRegistry->entries.erase(aliasRef); + userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(uri)); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command { - FlakeAlias flakeAlias; + FlakeUri alias; std::string name() override { @@ -189,21 +185,21 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { - expectArg("flake-id", &flakeAlias); + expectArg("alias", &alias); } void run() override { Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - userRegistry->entries.erase(flakeAlias); + userRegistry->entries.erase(FlakeRef(alias)); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs { - FlakeAlias flakeAlias; + FlakeUri alias; std::string name() override { @@ -217,7 +213,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { - expectArg("flake-id", &flakeAlias); + expectArg("alias", &alias); } void run(nix::ref store) override @@ -226,14 +222,13 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs Path userRegistryPath = getUserRegistryPath(); FlakeRegistry userRegistry = *readRegistry(userRegistryPath); - auto it = userRegistry.entries.find(flakeAlias); + auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - FlakeRef oldRef = it->second.ref; - it->second.ref = getFlake(*evalState, oldRef, true).ref; + it->second = getFlake(*evalState, it->second, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else - throw Error("the flake alias '%s' does not exist in the user registry", flakeAlias); + throw Error("the flake alias '%s' does not exist in the user registry", alias); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index e792ce96d..13a68a797 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,7 +7,6 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" -#include "primops/flake.hh" #include