Git: Use unified caching system

src/libstore/fetchers/git.cc

@@ -1,5 +1,6 @@
-#include "fetchers.hh"
+#include "fetchers/fetchers.hh"
-#include "parse.hh"
+#include "fetchers/cache.hh"
+#include "fetchers/parse.hh"
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
@@ -7,77 +8,15 @@
 
 #include <sys/time.h>
 
-#include <nlohmann/json.hpp>
-
 using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-static Path getCacheInfoPathFor(const std::string & name, const Hash & rev)
-{
-    Path cacheDir = getCacheDir() + "/nix/git-revs-v2";
-    std::string linkName =
-        name == "source"
-        ? rev.gitRev()
-        : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false);
-    return cacheDir + "/" + linkName + ".link";
-}
-
 static std::string readHead(const Path & path)
 {
     return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
 }
 
-static void cacheGitInfo(
-    Store & store,
-    const std::string & name,
-    const Tree & tree,
-    const Hash & rev)
-{
-    if (!tree.info.revCount || !tree.info.lastModified) return;
-
-    nlohmann::json json;
-    json["storePath"] = store.printStorePath(tree.storePath);
-    json["name"] = name;
-    json["rev"] = rev.gitRev();
-    json["revCount"] = *tree.info.revCount;
-    json["lastModified"] = *tree.info.lastModified;
-
-    auto cacheInfoPath = getCacheInfoPathFor(name, rev);
-    createDirs(dirOf(cacheInfoPath));
-    writeFile(cacheInfoPath, json.dump());
-}
-
-static std::optional<std::pair<Hash, Tree>> lookupGitInfo(
-    ref<Store> store,
-    const std::string & name,
-    const Hash & rev)
-{
-    try {
-        auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev)));
-
-        assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev);
-
-        auto storePath = store->parseStorePath((std::string) json["storePath"]);
-
-        if (store->isValidPath(storePath)) {
-            return {{rev, Tree{
-                .actualPath = store->toRealPath(storePath),
-                .storePath = std::move(storePath),
-                .info = TreeInfo {
-                    .revCount = json["revCount"],
-                    .lastModified = json["lastModified"],
-                }
-            }}};
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    return {};
-}
-
 struct GitInput : Input
 {
     ParsedURL url;
@@ -207,11 +146,38 @@ struct GitInput : Input
 
         assert(!rev || rev->type == htSHA1);
 
+        auto cacheType = shallow ? "git-shallow" : "git";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", cacheType},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
+                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
+                    },
+                },
+                input
+            };
+        };
+
         if (rev) {
-            if (auto tree = lookupGitInfo(store, name, *rev)) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
-                input->rev = tree->first;
+                return makeResult(res->first, std::move(res->second));
-                return {std::move(tree->second), input};
-            }
         }
 
         auto [isLocal, actualUrl_] = getActualUrl();
@@ -290,6 +256,13 @@ struct GitInput : Input
 
         if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
 
+        Attrs mutableAttrs({
+            {"type", cacheType},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
         Path repoDir;
 
         if (isLocal) {
@@ -301,6 +274,14 @@ struct GitInput : Input
 
         } else {
 
+            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+                if (!rev || rev == rev2) {
+                    input->rev = rev2;
+                    return makeResult(res->first, std::move(res->second));
+                }
+            }
+
             Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
             repoDir = cacheDir;
 
@@ -368,16 +349,15 @@ struct GitInput : Input
         if (isShallow && !shallow)
             throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
 
-        if (auto tree = lookupGitInfo(store, name, *input->rev)) {
-            assert(*input->rev == tree->first);
-            if (shallow) tree->second.info.revCount.reset();
-            return {std::move(tree->second), input};
-        }
-
         // FIXME: check whether rev is an ancestor of ref.
 
         printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
 
+        /* Now that we know the ref, check again whether we have it in
+           the store. */
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
         // FIXME: should pipe this, or find some better way to extract a
         // revision.
         auto source = sinkToSource([&](Sink & sink) {
@@ -395,21 +375,31 @@ struct GitInput : Input
 
         auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
 
-        auto tree = Tree {
+        Attrs infoAttrs({
-            .actualPath = store->toRealPath(storePath),
+            {"rev", input->rev->gitRev()},
-            .storePath = std::move(storePath),
+            {"lastModified", lastModified},
-            .info = TreeInfo {
+        });
-                .revCount =
-                    !shallow
-                    ? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })))
-                    : std::nullopt,
-                .lastModified = lastModified
-            }
-        };
 
-        cacheGitInfo(*store, name, tree, *input->rev);
+        if (!shallow)
+            infoAttrs.insert_or_assign("revCount",
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
 
-        return {std::move(tree), input};
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
     }
 };
 

src/libstore/fetchers/mercurial.cc

@@ -174,7 +174,7 @@ struct MercurialInput : Input
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
             -> std::pair<Tree, std::shared_ptr<const Input>>
         {
-            input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1);
+            assert(input->rev);
             assert(!rev || rev == input->rev);
             return {
                 Tree{
@@ -203,8 +203,13 @@ struct MercurialInput : Input
             {"ref", *input->ref},
         });
 
-        if (auto res = getCache()->lookup(store, mutableAttrs))
+        if (auto res = getCache()->lookup(store, mutableAttrs)) {
-            return makeResult(res->first, std::move(res->second));
+            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+            if (!rev || rev == rev2) {
+                input->rev = rev2;
+                return makeResult(res->first, std::move(res->second));
+            }
+        }
 
         Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
 

tests/fetchGit.sh

@@ -11,7 +11,7 @@ repo=$TEST_ROOT/git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git* $TEST_ROOT/worktree $TEST_ROOT/shallow
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow
 
 git init $repo
 git -C $repo config user.email "foobar@example.com"
@@ -59,10 +59,10 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
 
 # Fetching with a explicit hash should succeed.
-path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
 
-path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
 [[ $(cat $path2/hello) = utrecht ]]
 
 mv ${repo}-tmp $repo
@@ -99,7 +99,7 @@ path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"
 # Committing should not affect the store path.
 git -C $repo commit -m 'Bla3' -a
 
-path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchGit file://$repo).outPath")
+path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
 [[ $path2 = $path4 ]]
 
 # tarball-ttl should be ignored if we specify a rev
@@ -137,11 +137,10 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
 
 
 # Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/gitv2
+rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH
+# Try again, but without 'git' on PATH. This should fail.
 NIX=$(command -v nix)
-# This should fail
 (! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
 
 # Try again, with 'git' available.  This should work.