Merge branch 'flakeRegistryMapsFromRef' into flakes

flake-registry.json

@@ -1,5 +1,4 @@
 {
-    "version": 1,
     "flakes": {
         "dwarffs": {
             "uri": "github:edolstra/dwarffs/flake"
@@ -7,5 +6,6 @@
         "nixpkgs": {
             "uri": "github:edolstra/nixpkgs/flake"
         }
-    }
+    },
+    "version": 1
 }

src/libexpr/primops/flake.cc

@@ -18,21 +18,18 @@ std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
 {
     auto registry = std::make_shared<FlakeRegistry>();
 
-    try {
+    if (!pathExists(path))
-        auto json = nlohmann::json::parse(readFile(path));
+        return std::make_shared<FlakeRegistry>();
 
-        auto version = json.value("version", 0);
+    auto json = nlohmann::json::parse(readFile(path));
-        if (version != 1)
-            throw Error("flake registry '%s' has unsupported version %d", path, version);
 
-        auto flakes = json["flakes"];
+    auto version = json.value("version", 0);
-        for (auto i = flakes.begin(); i != flakes.end(); ++i) {
+    if (version != 1)
-            FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))};
+        throw Error("flake registry '%s' has unsupported version %d", path, version);
-            registry->entries.emplace(i.key(), entry);
+
-        }
+    auto flakes = json["flakes"];
-    } catch (SysError & e) {
+    for (auto i = flakes.begin(); i != flakes.end(); ++i)
-        if (e.errNo != ENOENT) throw;
+        registry->entries.emplace(i.key(), FlakeRef(i->value("uri", "")));
-    }
 
     return registry;
 }
@@ -40,48 +37,137 @@ std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
 /* Write the registry or lock file to a file. */
 void writeRegistry(FlakeRegistry registry, Path path)
 {
-    nlohmann::json json = {};
+    nlohmann::json json;
     json["version"] = 1;
-    json["flakes"] = {};
+    for (auto elem : registry.entries)
-    for (auto elem : registry.entries) {
+        json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} };
-        json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} };
-    }
     createDirs(dirOf(path));
     writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file.
 }
 
+LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
+{
+    FlakeRef flakeRef(json["uri"]);
+    if (!flakeRef.isImmutable())
+        throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
+
+    LockFile::FlakeEntry entry(flakeRef);
+
+    auto nonFlakeRequires = json["nonFlakeRequires"];
+
+    for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) {
+        FlakeRef flakeRef(i->value("uri", ""));
+        if (!flakeRef.isImmutable())
+            throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
+        entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef);
+    }
+
+    auto requires = json["requires"];
+
+    for (auto i = requires.begin(); i != requires.end(); ++i)
+        entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
+
+    return entry;
+}
+
+LockFile readLockFile(const Path & path)
+{
+    LockFile lockFile;
+
+    if (!pathExists(path))
+        return lockFile;
+
+    auto json = nlohmann::json::parse(readFile(path));
+
+    auto version = json.value("version", 0);
+    if (version != 1)
+        throw Error("lock file '%s' has unsupported version %d", path, version);
+
+    auto nonFlakeRequires = json["nonFlakeRequires"];
+
+    for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) {
+        FlakeRef flakeRef(i->value("uri", ""));
+        if (!flakeRef.isImmutable())
+            throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
+        lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef);
+    }
+
+    auto requires = json["requires"];
+
+    for (auto i = requires.begin(); i != requires.end(); ++i)
+        lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
+
+    return lockFile;
+}
+
+nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry)
+{
+    nlohmann::json json;
+    json["uri"] = entry.ref.to_string();
+    for (auto & x : entry.nonFlakeEntries)
+        json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string();
+    for (auto & x : entry.flakeEntries)
+        json["requires"][x.first] = flakeEntryToJson(x.second);
+    return json;
+}
+
+void writeLockFile(LockFile lockFile, Path path)
+{
+    nlohmann::json json;
+    json["version"] = 1;
+    json["nonFlakeRequires"];
+    for (auto & x : lockFile.nonFlakeEntries)
+        json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string();
+    for (auto & x : lockFile.flakeEntries)
+        json["requires"][x.first] = flakeEntryToJson(x.second);
+    createDirs(dirOf(path));
+    writeFile(path, json.dump(4)); // '4' = indentation in json file
+}
+
+std::shared_ptr<FlakeRegistry> getGlobalRegistry()
+{
+    return std::make_shared<FlakeRegistry>();
+}
+
 Path getUserRegistryPath()
 {
     return getHome() + "/.config/nix/registry.json";
 }
 
-std::shared_ptr<FlakeRegistry> getGlobalRegistry()
-{
-    // FIXME: get from nixos.org.
-    Path registryFile = settings.nixDataDir + "/nix/flake-registry.json";
-    return readRegistry(registryFile);
-}
-
 std::shared_ptr<FlakeRegistry> getUserRegistry()
 {
     return readRegistry(getUserRegistryPath());
 }
 
+std::shared_ptr<FlakeRegistry> getLocalRegistry()
+{
+    Path registryFile = settings.nixDataDir + "/nix/flake-registry.json";
+    return readRegistry(registryFile);
+}
+
 std::shared_ptr<FlakeRegistry> getFlagRegistry()
 {
+    // TODO (Nick): Implement this.
     return std::make_shared<FlakeRegistry>();
-    // TODO: Implement this once the right flags are implemented.
 }
 
 const std::vector<std::shared_ptr<FlakeRegistry>> EvalState::getFlakeRegistries()
 {
     std::vector<std::shared_ptr<FlakeRegistry>> registries;
-    registries.push_back(getGlobalRegistry());
+    if (evalSettings.pureEval) {
-    registries.push_back(getUserRegistry());
+        registries.push_back(std::make_shared<FlakeRegistry>()); // global
+        registries.push_back(std::make_shared<FlakeRegistry>()); // user
+        registries.push_back(std::make_shared<FlakeRegistry>()); // local
+    } else {
+        registries.push_back(getGlobalRegistry());
+        registries.push_back(getUserRegistry());
+        registries.push_back(getLocalRegistry());
+    }
     registries.push_back(getFlagRegistry());
     return registries;
 }
 
+// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`.
 Value * makeFlakeRegistryValue(EvalState & state)
 {
     auto v = state.allocValue();
@@ -95,9 +181,9 @@ Value * makeFlakeRegistryValue(EvalState & state)
 
     for (auto & registry : registries) {
         for (auto & entry : registry->entries) {
-            auto vEntry = state.allocAttr(*v, entry.first);
+            auto vEntry = state.allocAttr(*v, entry.first.to_string());
             state.mkAttrs(*vEntry, 2);
-            mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string());
+            mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.to_string());
             vEntry->attrs->sort();
         }
     }
@@ -108,21 +194,30 @@ Value * makeFlakeRegistryValue(EvalState & state)
 }
 
 static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef,
-    std::vector<std::shared_ptr<FlakeRegistry>> registries)
+    std::vector<std::shared_ptr<FlakeRegistry>> registries, std::vector<FlakeRef> pastSearches = {})
 {
-    if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data)) {
+    for (std::shared_ptr<FlakeRegistry> registry : registries) {
-        for (auto registry : registries) {
+        auto i = registry->entries.find(flakeRef);
-            auto i = registry->entries.find(refData->id);
+        if (i != registry->entries.end()) {
-            if (i != registry->entries.end()) {
+            auto newRef = i->second;
-                auto newRef = FlakeRef(i->second.ref);
+            if (std::get_if<FlakeRef::IsAlias>(&flakeRef.data)) {
-                if (!newRef.isDirect())
+                if (flakeRef.ref) newRef.ref = flakeRef.ref;
-                    throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string());
+                if (flakeRef.rev) newRef.rev = flakeRef.rev;
-                return newRef;
             }
+            std::string errorMsg = "found cycle in flake registries: ";
+            for (FlakeRef oldRef : pastSearches) {
+                errorMsg += oldRef.to_string();
+                if (oldRef == newRef)
+                    throw Error(errorMsg);
+                errorMsg += " - ";
+            }
+            pastSearches.push_back(newRef);
+            return lookupFlake(state, newRef, registries, pastSearches);
         }
-        throw Error("cannot find flake '%s' in the flake registry or in the flake lock file", refData->id);
+    }
-    } else
+    if (!flakeRef.isDirect())
-        return flakeRef;
+        throw Error("indirect flake URI '%s' is the result of a lookup", flakeRef.to_string());
+    return flakeRef;
 }
 
 struct FlakeSourceInfo
@@ -132,17 +227,14 @@ struct FlakeSourceInfo
     std::optional<uint64_t> revCount;
 };
 
-static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
+static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false)
 {
-    FlakeRef directFlakeRef = FlakeRef(flakeRef);
+    FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries());
-    if (!flakeRef.isDirect()) {
-        directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries());
-    }
-    assert(directFlakeRef.isDirect());
-    // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes..
 
-    if (auto refData = std::get_if<FlakeRef::IsGitHub>(&directFlakeRef.data)) {
+    // This only downloads only one revision of the repo, not the entire history.
-        // FIXME: require hash in pure mode.
+    if (auto refData = std::get_if<FlakeRef::IsGitHub>(&fRef.data)) {
+        if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable())
+            throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string());
 
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
@@ -151,14 +243,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
 
         auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
             refData->owner, refData->repo,
-            refData->rev
+            fRef.rev ? fRef.rev->to_string(Base16, false)
-                ? refData->rev->to_string(Base16, false)
+                : fRef.ref ? *fRef.ref : "master");
-                : refData->ref
-                  ? *refData->ref
-                  : "master");
 
         auto result = getDownloader()->downloadCached(state.store, url, true, "source",
-            Hash(), nullptr, refData->rev ? 1000000000 : settings.tarballTtl);
+            Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl);
 
         if (!result.etag)
             throw Error("did not receive an ETag header from '%s'", url);
@@ -173,9 +262,10 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
         return info;
     }
 
-    else if (auto refData = std::get_if<FlakeRef::IsGit>(&directFlakeRef.data)) {
+    // This downloads the entire git history
-        auto gitInfo = exportGit(state.store, refData->uri, refData->ref,
+    else if (auto refData = std::get_if<FlakeRef::IsGit>(&fRef.data)) {
-            refData->rev ? refData->rev->to_string(Base16, false) : "", "source");
+        auto gitInfo = exportGit(state.store, refData->uri, fRef.ref,
+            fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source");
         FlakeSourceInfo info;
         info.storePath = gitInfo.storePath;
         info.rev = Hash(gitInfo.rev, htSHA1);
@@ -183,7 +273,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
         return info;
     }
 
-    else if (auto refData = std::get_if<FlakeRef::IsPath>(&directFlakeRef.data)) {
+    else if (auto refData = std::get_if<FlakeRef::IsPath>(&fRef.data)) {
         if (!pathExists(refData->path + "/.git"))
             throw Error("flake '%s' does not reference a Git repository", refData->path);
         auto gitInfo = exportGit(state.store, refData->path, {}, "", "source");
@@ -197,9 +287,10 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
     else abort();
 }
 
-Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
+// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function.
+Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
 {
-    auto sourceInfo = fetchFlake(state, flakeRef);
+    FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
     debug("got flake source '%s' with revision %s",
         sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
 
@@ -209,16 +300,13 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
     if (state.allowedPaths)
         state.allowedPaths->insert(flakePath);
 
-    FlakeRef newFlakeRef(flakeRef);
+    Flake flake(flakeRef);
-    if (std::get_if<FlakeRef::IsGitHub>(&newFlakeRef.data)) {
+    if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
-        FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef);
+        if (sourceInfo.rev)
-        if (srcInfo.rev) {
+            flake.ref = FlakeRef(flakeRef.baseRef().to_string()
-            std::string uri = flakeRef.baseRef().to_string();
+                + "/" + sourceInfo.rev->to_string(Base16, false));
-            newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string(Base16, false));
-        }
     }
 
-    Flake flake(newFlakeRef);
     flake.path = flakePath;
     flake.revCount = sourceInfo.revCount;
 
@@ -242,141 +330,142 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
                 *(**requires).value->listElems()[n], *(**requires).pos)));
     }
 
+    if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) {
+        state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos);
+        for (Attr attr : *(*(**nonFlakeRequires).value).attrs) {
+            std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos);
+            FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri);
+            flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef);
+        }
+    }
+
     if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
         state.forceFunction(*(**provides).value, *(**provides).pos);
         flake.vProvides = (**provides).value;
     } else
         throw Error("flake lacks attribute 'provides'");
 
-    auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack
+    const Path lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack
 
-    if (pathExists(lockFile)) {
+    flake.lockFile = readLockFile(lockFile);
-        flake.lockFile = readRegistry(lockFile);
-        for (auto & entry : flake.lockFile->entries)
-            if (!entry.second.ref.isImmutable())
-                throw Error("flake lock file '%s' contains mutable entry '%s'",
-                    lockFile, entry.second.ref.to_string());
-    }
 
     return flake;
 }
 
+// Get the `NonFlake` corresponding to a `FlakeRef`.
+NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias)
+{
+    FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
+    debug("got non-flake source '%s' with revision %s",
+        sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
+
+    auto flakePath = sourceInfo.storePath;
+    state.store->assertStorePath(flakePath);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(flakePath);
+
+    NonFlake nonFlake(flakeRef);
+    if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
+        if (sourceInfo.rev)
+            nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string()
+                + "/" + sourceInfo.rev->to_string(Base16, false));
+    }
+
+    nonFlake.path = flakePath;
+
+    nonFlake.alias = alias;
+
+    return nonFlake;
+}
+
 /* Given a flake reference, recursively fetch it and its
    dependencies.
    FIXME: this should return a graph of flakes.
 */
-static std::tuple<FlakeId, std::map<FlakeId, Flake>> resolveFlake(EvalState & state,
+Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake)
-    const FlakeRef & topRef, bool impureTopRef)
 {
-    std::map<FlakeId, Flake> done;
+    Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef);
-    std::queue<std::tuple<FlakeRef, bool>> todo;
+    Dependencies deps(flake);
-    std::optional<FlakeId> topFlakeId; /// FIXME: ambiguous
-    todo.push({topRef, true});
 
-    auto registries = state.getFlakeRegistries();
+    for (auto & nonFlakeInfo : flake.nonFlakeRequires)
-    //std::shared_ptr<FlakeRegistry> localRegistry = registries.at(2);
+        deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first));
 
-    while (!todo.empty()) {
+    for (auto & newFlakeRef : flake.requires)
-        auto [flakeRef, toplevel] = todo.front();
+        deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, false));
-        todo.pop();
 
-        if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data)) {
+    return deps;
-            if (done.count(refData->id)) continue; // optimization
-            flakeRef = lookupFlake(state, flakeRef,
-                !evalSettings.pureEval || (toplevel && impureTopRef) ? registries : std::vector<std::shared_ptr<FlakeRegistry>>());
-            // This is why we need the `registries`.
-        }
-
-#if 0
-        if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef))
-            throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string());
-#endif
-
-        auto flake = getFlake(state, flakeRef);
-
-        if (done.count(flake.id)) continue;
-
-        if (toplevel) topFlakeId = flake.id;
-
-        for (auto & require : flake.requires)
-            todo.push({require, false});
-
-#if 0
-        // The following piece of code basically adds the FlakeRefs from
-        // the lockfiles of dependencies to the localRegistry. This is used
-        // to resolve future `FlakeId`s, in `lookupFlake` a bit above this.
-        if (flake.lockFile)
-            for (auto & entry : flake.lockFile->entries) {
-                if (localRegistry->entries.count(entry.first)) continue;
-                localRegistry->entries.emplace(entry.first, entry.second);
-            }
-#endif
-
-        done.emplace(flake.id, std::move(flake));
-    }
-
-    assert(topFlakeId);
-    return {*topFlakeId, std::move(done)};
 }
 
-FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef)
+LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps)
 {
-    FlakeRegistry newLockFile;
+    LockFile::FlakeEntry entry(deps.flake.ref);
-    std::map<FlakeId, Flake> myDependencyMap = get<1>(resolveFlake(evalState, flakeRef, false));
+
-    // Nick assumed that "topRefPure" means that the Flake for flakeRef can be
+    for (Dependencies & deps : deps.flakeDeps)
-    // fetched purely.
+        entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps));
-    for (auto const& require : myDependencyMap) {
+
-        FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.second.ref);
+    for (NonFlake & nonFlake : deps.nonFlakeDeps)
-        // The FlakeRefs are immutable because they come out of the Flake objects,
+        entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref);
-        // not from the requires.
+
-        newLockFile.entries.insert(std::pair<FlakeId, FlakeRegistry::Entry>(require.first, entry));
+    return entry;
-    }
-    return newLockFile;
 }
 
-void updateLockFile(EvalState & state, std::string path)
+LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef)
+{
+    Dependencies deps = resolveFlake(evalState, flakeRef, true);
+    LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps);
+    LockFile lockFile;
+    lockFile.flakeEntries = entry.flakeEntries;
+    lockFile.nonFlakeEntries = entry.nonFlakeEntries;
+    return lockFile;
+}
+
+void updateLockFile(EvalState & state, Path path)
 {
     // 'path' is the path to the local flake repo.
     FlakeRef flakeRef = FlakeRef("file://" + path);
     if (std::get_if<FlakeRef::IsGit>(&flakeRef.data)) {
-        FlakeRegistry newLockFile = updateLockFile(state, flakeRef);
+        LockFile lockFile = getLockFile(state, flakeRef);
-        writeRegistry(newLockFile, path + "/flake.lock");
+        writeLockFile(lockFile, path + "/flake.lock");
     } else if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
         throw UsageError("you can only update local flakes, not flakes on GitHub");
     } else {
-        throw UsageError("you can only update local flakes, not flakes through their FlakeId");
+        throw UsageError("you can only update local flakes, not flakes through their FlakeAlias");
     }
 }
 
+// Return the `provides` of the top flake, while assigning to `v` the provides
+// of the dependencies as well.
 Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v)
 {
-    auto [topFlakeId, flakes] = resolveFlake(state, flakeRef, impureTopRef);
+    Dependencies deps = resolveFlake(state, flakeRef, impureTopRef);
 
     // FIXME: we should call each flake with only its dependencies
     // (rather than the closure of the top-level flake).
 
     auto vResult = state.allocValue();
+    // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`.
 
-    state.mkAttrs(*vResult, flakes.size());
+    state.mkAttrs(*vResult, deps.flakeDeps.size());
 
-    Value * vTop = 0;
+    Value * vTop = state.allocAttr(*vResult, deps.flake.id);
 
-    for (auto & flake : flakes) {
+    for (auto & dep : deps.flakeDeps) {
-        auto vFlake = state.allocAttr(*vResult, flake.second.id);
+        Flake flake = dep.flake;
-        if (topFlakeId == flake.second.id) vTop = vFlake;
+        auto vFlake = state.allocAttr(*vResult, flake.id);
 
         state.mkAttrs(*vFlake, 4);
 
-        mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description);
+        mkString(*state.allocAttr(*vFlake, state.sDescription), flake.description);
 
-        state.store->assertStorePath(flake.second.path);
+        state.store->assertStorePath(flake.path);
-        mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.second.path, {flake.second.path});
+        mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path});
 
-        if (flake.second.revCount)
+        if (flake.revCount)
-            mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.second.revCount);
+            mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount);
 
         auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides"));
-        mkApp(*vProvides, *flake.second.vProvides, *vResult);
+        mkApp(*vProvides, *flake.vProvides, *vResult);
 
         vFlake->attrs->sort();
     }
@@ -389,6 +478,7 @@ Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impure
     return vTop;
 }
 
+// This function is exposed to be used in nix files.
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v);

src/libexpr/primops/flake.hh

@@ -10,13 +10,21 @@ class EvalState;
 
 struct FlakeRegistry
 {
-    struct Entry
+    std::map<FlakeRef, FlakeRef> entries;
+};
+
+struct LockFile
+{
+    struct FlakeEntry
     {
         FlakeRef ref;
-        Entry(const FlakeRef & flakeRef) : ref(flakeRef) {};
+        std::map<FlakeId, FlakeEntry> flakeEntries;
-        Entry operator=(const Entry & entry) { return Entry(entry.ref); }
+        std::map<FlakeId, FlakeRef> nonFlakeEntries;
+        FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {};
     };
-    std::map<FlakeId, Entry> entries;
+
+    std::map<FlakeId, FlakeEntry> flakeEntries;
+    std::map<FlakeId, FlakeRef> nonFlakeEntries;
 };
 
 Path getUserRegistryPath();
@@ -37,17 +45,37 @@ struct Flake
     Path path;
     std::optional<uint64_t> revCount;
     std::vector<FlakeRef> requires;
-    std::shared_ptr<FlakeRegistry> lockFile;
+    LockFile lockFile;
+    std::map<FlakeAlias, FlakeRef> nonFlakeRequires;
     Value * vProvides; // FIXME: gc
-    // commit hash
     // date
     // content hash
-    Flake(FlakeRef & flakeRef) : ref(flakeRef) {};
+    Flake(const FlakeRef flakeRef) : ref(flakeRef) {};
 };
 
-Flake getFlake(EvalState &, const FlakeRef &);
+struct NonFlake
+{
+    FlakeAlias alias;
+    FlakeRef ref;
+    Path path;
+    // date
+    // content hash
+    NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {};
+};
+
+Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
+
+struct Dependencies
+{
+    Flake flake;
+    std::vector<Dependencies> flakeDeps; // The flake dependencies
+    std::vector<NonFlake> nonFlakeDeps;
+    Dependencies(const Flake & flake) : flake(flake) {}
+};
+
+Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake = true);
 
 FlakeRegistry updateLockFile(EvalState &, Flake &);
 
-void updateLockFile(EvalState &, std::string);
+void updateLockFile(EvalState &, Path path);
 }

src/libexpr/primops/flakeref.cc

@@ -19,7 +19,7 @@ const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex +
 // "master/e72daba8250068216d79d2aeef40d4d95aff6666").
 const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))";
 
-const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*";
+const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*";
 
 // GitHub references.
 const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*";
@@ -37,7 +37,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
     // FIXME: could combine this into one regex.
 
     static std::regex flakeRegex(
-        "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?",
+        "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?",
         std::regex::ECMAScript);
 
     static std::regex githubRegex(
@@ -55,14 +55,14 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
 
     std::cmatch match;
     if (std::regex_match(uri.c_str(), match, flakeRegex)) {
-        IsFlakeId d;
+        IsAlias d;
-        d.id = match[1];
+        d.alias = match[1];
         if (match[2].matched)
-            d.rev = Hash(match[2], htSHA1);
+            rev = Hash(match[2], htSHA1);
         else if (match[3].matched) {
-            d.ref = match[3];
+            ref = match[3];
             if (match[4].matched)
-                d.rev = Hash(match[4], htSHA1);
+                rev = Hash(match[4], htSHA1);
         }
         data = d;
     }
@@ -72,9 +72,9 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
         d.owner = match[1];
         d.repo = match[2];
         if (match[3].matched)
-            d.rev = Hash(match[3], htSHA1);
+            rev = Hash(match[3], htSHA1);
         else if (match[4].matched) {
-            d.ref = match[4];
+            ref = match[4];
         }
         data = d;
     }
@@ -92,16 +92,16 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
             if (name == "rev") {
                 if (!std::regex_match(value, revRegex))
                     throw Error("invalid Git revision '%s'", value);
-                d.rev = Hash(value, htSHA1);
+                rev = Hash(value, htSHA1);
             } else if (name == "ref") {
                 if (!std::regex_match(value, refRegex2))
                     throw Error("invalid Git ref '%s'", value);
-                d.ref = value;
+                ref = value;
             } else
                 // FIXME: should probably pass through unknown parameters
                 throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
         }
-        if (d.rev && !d.ref)
+        if (rev && !ref)
             throw Error("flake URI '%s' lacks a Git ref", uri);
         data = d;
     }
@@ -118,66 +118,40 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
 
 std::string FlakeRef::to_string() const
 {
-    if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&data)) {
+    std::string string;
-        return
+    if (auto refData = std::get_if<FlakeRef::IsAlias>(&data))
-            "flake:" + refData->id +
+        string = "flake:" + refData->alias;
-            (refData->ref ? "/" + *refData->ref : "") +
-            (refData->rev ? "/" + refData->rev->to_string(Base16, false) : "");
-    }
 
     else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) {
-        assert(!refData->ref || !refData->rev);
+        assert(!ref || !rev);
-        return
+        string = "github:" + refData->owner + "/" + refData->repo;
-            "github:" + refData->owner + "/" + refData->repo +
-            (refData->ref ? "/" + *refData->ref : "") +
-            (refData->rev ? "/" + refData->rev->to_string(Base16, false) : "");
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) {
-        assert(refData->ref || !refData->rev);
+        assert(ref || !rev);
-        return
+        string = refData->uri;
-            refData->uri +
-            (refData->ref ? "?ref=" + *refData->ref : "") +
-            (refData->rev ? "&rev=" + refData->rev->to_string(Base16, false) : "");
     }
 
-    else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) {
+    else if (auto refData = std::get_if<FlakeRef::IsPath>(&data))
         return refData->path;
-    }
 
     else abort();
+
+    string += (ref ? "/" + *ref : "") +
+              (rev ? "/" + rev->to_string(Base16, false) : "");
+    return string;
 }
 
 bool FlakeRef::isImmutable() const
 {
-    if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&data))
+    return (bool) rev;
-        return (bool) refData->rev;
-
-    else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data))
-        return (bool) refData->rev;
-
-    else if (auto refData = std::get_if<FlakeRef::IsGit>(&data))
-        return (bool) refData->rev;
-
-    else if (std::get_if<FlakeRef::IsPath>(&data))
-        return false;
-
-    else abort();
 }
 
 FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef.
 {
     FlakeRef result(*this);
-    if (auto refData = std::get_if<FlakeRef::IsGitHub>(&result.data)) {
+    result.ref = std::nullopt;
-        refData->ref = std::nullopt;
+    result.rev = std::nullopt;
-        refData->rev = std::nullopt;
-    } else if (auto refData = std::get_if<FlakeRef::IsGit>(&result.data)) {
-        refData->ref = std::nullopt;
-        refData->rev = std::nullopt;
-    } else if (auto refData = std::get_if<FlakeRef::IsGit>(&result.data)) {
-        refData->ref = std::nullopt;
-        refData->rev = std::nullopt;
-    }
     return result;
 }
 }

src/libexpr/primops/flakeref.hh

@@ -98,53 +98,65 @@ namespace nix {
 */
 
 typedef std::string FlakeId;
+typedef std::string FlakeAlias;
+typedef std::string FlakeUri;
 
 struct FlakeRef
 {
-    struct IsFlakeId
+    struct IsAlias
     {
-        FlakeId id;
+        FlakeAlias alias;
-        std::optional<std::string> ref;
+        bool operator<(const IsAlias & b) const { return alias < b.alias; };
-        std::optional<Hash> rev;
+        bool operator==(const IsAlias & b) const { return alias == b.alias; };
     };
 
-    struct IsGitHub
+    struct IsGitHub {
-    {
         std::string owner, repo;
-        std::optional<std::string> ref;
+        bool operator<(const IsGitHub & b) const {
-        std::optional<Hash> rev;
+            return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo);
+        }
+        bool operator==(const IsGitHub & b) const {
+            return owner == b.owner && repo == b.repo;
+        }
     };
 
+    // Git, Tarball
     struct IsGit
     {
         std::string uri;
-        std::optional<std::string> ref;
+        bool operator<(const IsGit & b) const { return uri < b.uri; }
-        std::optional<Hash> rev;
+        bool operator==(const IsGit & b) const { return uri == b.uri; }
     };
 
     struct IsPath
     {
         Path path;
+        bool operator<(const IsPath & b) const { return path < b.path; }
+        bool operator==(const IsPath & b) const { return path == b.path; }
     };
 
     // Git, Tarball
 
-    std::variant<IsFlakeId, IsGitHub, IsGit, IsPath> data;
+    std::variant<IsAlias, IsGitHub, IsGit, IsPath> data;
+
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    bool operator<(const FlakeRef & flakeRef) const
+    {
+        return std::make_tuple(this->data, ref, rev) <
+            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev);
+    }
+
+    bool operator==(const FlakeRef & flakeRef) const
+    {
+        return std::make_tuple(this->data, ref, rev) ==
+            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev);
+    }
 
     // Parse a flake URI.
     FlakeRef(const std::string & uri, bool allowRelative = false);
 
-    // Default constructor
-    FlakeRef(const FlakeRef & flakeRef) : data(flakeRef.data) {};
-
-    /* Unify two flake references so that the resulting reference
-       combines the information from both. For example,
-       "nixpkgs/<hash>" and "github:NixOS/nixpkgs" unifies to
-       "nixpkgs/master". May throw an exception if the references are
-       incompatible (e.g. "nixpkgs/<hash1>" and "nixpkgs/<hash2>",
-       where hash1 != hash2). */
-    FlakeRef(const FlakeRef & a, const FlakeRef & b);
-
     // FIXME: change to operator <<.
     std::string to_string() const;
 
@@ -152,7 +164,7 @@ struct FlakeRef
        a flake ID, which requires a lookup in the flake registry. */
     bool isDirect() const
     {
-        return !std::get_if<FlakeRef::IsFlakeId>(&data);
+        return !std::get_if<FlakeRef::IsAlias>(&data);
     }
 
     /* Check whether this is an "immutable" flake reference, that is,

src/nix/build.cc

@@ -1,4 +1,3 @@
-#include "primops/flake.hh"
 #include "eval.hh"
 #include "command.hh"
 #include "common-args.hh"
@@ -11,7 +10,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 {
     Path outLink = "result";
 
-    std::optional<std::string> gitRepo = std::nullopt;
+    bool updateLock = true;
 
     CmdBuild()
     {
@@ -28,9 +27,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand
             .set(&outLink, Path(""));
 
         mkFlag()
-            .longName("update-lock-file")
+            .longName("no-update")
-            .description("update the lock file")
+            .description("don't update the lock file")
-            .dest(&gitRepo);
+            .set(&updateLock, false);
     }
 
     std::string name() override
@@ -78,8 +77,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand
                     }
         }
 
-        if (gitRepo)
+        // FlakeUri flakeUri = "";
-            updateLockFile(*evalState, *gitRepo);
+        // if(updateLock)
+        //     for (uint i = 0; i < installables.size(); i++)
+        //         // if (auto flakeUri = installableToFlakeUri)
+        //             updateLockFile(*evalState, flakeUri);
     }
 };
 

src/nix/command.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "args.hh"
+#include "primops/flake.hh"
 #include "common-eval-args.hh"
 
 namespace nix {
@@ -46,7 +47,7 @@ struct GitRepoCommand : virtual Args
 
 struct FlakeCommand : virtual Args
 {
-    std::string flakeUri;
+    FlakeUri flakeUri;
 
     FlakeCommand()
     {

src/nix/flake.cc

@@ -1,10 +1,10 @@
-#include "primops/flake.hh"
 #include "command.hh"
 #include "common-args.hh"
 #include "shared.hh"
 #include "progress-bar.hh"
 #include "eval.hh"
 #include <nlohmann/json.hpp>
+#include <queue>
 
 using namespace nix;
 
@@ -28,10 +28,70 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs
 
         stopProgressBar();
 
-        for (auto & registry : registries) {
+        for (auto & registry : registries)
-            for (auto & entry : registry->entries) {
+            for (auto & entry : registry->entries)
-                std::cout << entry.first << " " << entry.second.ref.to_string() << "\n";
+                std::cout << entry.first.to_string() << " " << entry.second.to_string() << "\n";
-            }
+    }
+};
+
+void printFlakeInfo(Flake & flake, bool json) {
+    if (json) {
+        nlohmann::json j;
+        j["id"] = flake.id;
+        j["location"] = flake.path;
+        j["description"] = flake.description;
+        std::cout << j.dump(4) << std::endl;
+    } else {
+        std::cout << "ID:          " << flake.id << "\n";
+        std::cout << "Description: " << flake.description << "\n";
+        std::cout << "Location:    " << flake.path << "\n";
+    }
+}
+
+void printNonFlakeInfo(NonFlake & nonFlake, bool json) {
+    if (json) {
+        nlohmann::json j;
+        j["name"] = nonFlake.alias;
+        j["location"] = nonFlake.path;
+        std::cout << j.dump(4) << std::endl;
+    } else {
+        std::cout << "name:        " << nonFlake.alias << "\n";
+        std::cout << "Location:    " << nonFlake.path << "\n";
+    }
+}
+
+struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs
+{
+    std::string name() override
+    {
+        return "deps";
+    }
+
+    std::string description() override
+    {
+        return "list informaton about dependencies";
+    }
+
+    void run(nix::ref<nix::Store> store) override
+    {
+        auto evalState = std::make_shared<EvalState>(searchPath, store);
+
+        FlakeRef flakeRef(flakeUri);
+
+        Dependencies deps = resolveFlake(*evalState, flakeRef, true);
+
+        std::queue<Dependencies> todo;
+        todo.push(deps);
+
+        while (!todo.empty()) {
+            deps = todo.front();
+            todo.pop();
+
+            for (auto & nonFlake : deps.nonFlakeDeps)
+                printNonFlakeInfo(nonFlake, json);
+
+            for (auto & newDeps : deps.flakeDeps)
+                todo.push(newDeps);
         }
     }
 };
@@ -72,23 +132,15 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand
     void run(nix::ref<nix::Store> store) override
     {
         auto evalState = std::make_shared<EvalState>(searchPath, store);
-        nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri));
+        nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true);
-        if (json) {
+        printFlakeInfo(flake, json);
-            nlohmann::json j;
-            j["location"] = flake.path;
-            j["description"] = flake.description;
-            std::cout << j.dump(4) << std::endl;
-        } else {
-            std::cout << "Description: " << flake.description << "\n";
-            std::cout << "Location:    " << flake.path << "\n";
-        }
     }
 };
 
 struct CmdFlakeAdd : MixEvalArgs, Command
 {
-    std::string flakeId;
+    FlakeUri alias;
-    std::string flakeUri;
+    FlakeUri uri;
 
     std::string name() override
     {
@@ -102,25 +154,24 @@ struct CmdFlakeAdd : MixEvalArgs, Command
 
     CmdFlakeAdd()
     {
-        expectArg("flake-id", &flakeId);
+        expectArg("alias", &alias);
-        expectArg("flake-uri", &flakeUri);
+        expectArg("flake-uri", &uri);
     }
 
     void run() override
     {
-        FlakeRef newFlakeRef(flakeUri);
+        FlakeRef aliasRef(alias);
         Path userRegistryPath = getUserRegistryPath();
         auto userRegistry = readRegistry(userRegistryPath);
-        FlakeRegistry::Entry entry(newFlakeRef);
+        userRegistry->entries.erase(aliasRef);
-        userRegistry->entries.erase(flakeId);
+        userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(uri));
-        userRegistry->entries.insert_or_assign(flakeId, newFlakeRef);
         writeRegistry(*userRegistry, userRegistryPath);
     }
 };
 
 struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command
 {
-    std::string flakeId;
+    FlakeUri alias;
 
     std::string name() override
     {
@@ -134,21 +185,21 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command
 
     CmdFlakeRemove()
     {
-        expectArg("flake-id", &flakeId);
+        expectArg("alias", &alias);
     }
 
     void run() override
     {
         Path userRegistryPath = getUserRegistryPath();
         auto userRegistry = readRegistry(userRegistryPath);
-        userRegistry->entries.erase(flakeId);
+        userRegistry->entries.erase(FlakeRef(alias));
         writeRegistry(*userRegistry, userRegistryPath);
     }
 };
 
 struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs
 {
-    std::string flakeId;
+    FlakeUri alias;
 
     std::string name() override
     {
@@ -162,7 +213,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs
 
     CmdFlakePin()
     {
-        expectArg("flake-id", &flakeId);
+        expectArg("alias", &alias);
     }
 
     void run(nix::ref<nix::Store> store) override
@@ -171,14 +222,13 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs
 
         Path userRegistryPath = getUserRegistryPath();
         FlakeRegistry userRegistry = *readRegistry(userRegistryPath);
-        auto it = userRegistry.entries.find(flakeId);
+        auto it = userRegistry.entries.find(FlakeRef(alias));
         if (it != userRegistry.entries.end()) {
-            FlakeRef oldRef = it->second.ref;
+            it->second = getFlake(*evalState, it->second, true).ref;
-            it->second.ref = getFlake(*evalState, oldRef).ref;
             // The 'ref' in 'flake' is immutable.
             writeRegistry(userRegistry, userRegistryPath);
         } else
-            throw Error("the flake identifier '%s' does not exist in the user registry", flakeId);
+            throw Error("the flake alias '%s' does not exist in the user registry", alias);
     }
 };
 
@@ -218,6 +268,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command
         : MultiCommand({make_ref<CmdFlakeList>()
             , make_ref<CmdFlakeUpdate>()
             , make_ref<CmdFlakeInfo>()
+            , make_ref<CmdFlakeDeps>()
             , make_ref<CmdFlakeAdd>()
             , make_ref<CmdFlakeRemove>()
             , make_ref<CmdFlakePin>()

src/nix/installables.cc

@@ -7,7 +7,6 @@
 #include "get-drvs.hh"
 #include "store-api.hh"
 #include "shared.hh"
-#include "primops/flake.hh"
 
 #include <regex>