From 38e360154d8ef6a70e3101a9b0d850e63fbfdfda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Mar 2020 22:32:26 +0100 Subject: [PATCH] Git: Use unified caching system --- src/libstore/fetchers/git.cc | 164 ++++++++++++++--------------- src/libstore/fetchers/mercurial.cc | 11 +- tests/fetchGit.sh | 13 ++- 3 files changed, 91 insertions(+), 97 deletions(-) diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index c5d4f019c..c1044d5a2 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -1,5 +1,6 @@ -#include "fetchers.hh" -#include "parse.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/cache.hh" +#include "fetchers/parse.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" @@ -7,77 +8,15 @@ #include -#include - using namespace std::string_literals; namespace nix::fetchers { -static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) -{ - Path cacheDir = getCacheDir() + "/nix/git-revs-v2"; - std::string linkName = - name == "source" - ? rev.gitRev() - : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false); - return cacheDir + "/" + linkName + ".link"; -} - static std::string readHead(const Path & path) { return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); } -static void cacheGitInfo( - Store & store, - const std::string & name, - const Tree & tree, - const Hash & rev) -{ - if (!tree.info.revCount || !tree.info.lastModified) return; - - nlohmann::json json; - json["storePath"] = store.printStorePath(tree.storePath); - json["name"] = name; - json["rev"] = rev.gitRev(); - json["revCount"] = *tree.info.revCount; - json["lastModified"] = *tree.info.lastModified; - - auto cacheInfoPath = getCacheInfoPathFor(name, rev); - createDirs(dirOf(cacheInfoPath)); - writeFile(cacheInfoPath, json.dump()); -} - -static std::optional> lookupGitInfo( - ref store, - const std::string & name, - const Hash & rev) -{ - try { - auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev))); - - assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev); - - auto storePath = store->parseStorePath((std::string) json["storePath"]); - - if (store->isValidPath(storePath)) { - return {{rev, Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = json["revCount"], - .lastModified = json["lastModified"], - } - }}}; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - return {}; -} - struct GitInput : Input { ParsedURL url; @@ -207,11 +146,38 @@ struct GitInput : Input assert(!rev || rev->type == htSHA1); + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + if (rev) { - if (auto tree = lookupGitInfo(store, name, *rev)) { - input->rev = tree->first; - return {std::move(tree->second), input}; - } + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); } auto [isLocal, actualUrl_] = getActualUrl(); @@ -290,6 +256,13 @@ struct GitInput : Input if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + Path repoDir; if (isLocal) { @@ -301,6 +274,14 @@ struct GitInput : Input } else { + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; @@ -368,16 +349,15 @@ struct GitInput : Input if (isShallow && !shallow) throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); - if (auto tree = lookupGitInfo(store, name, *input->rev)) { - assert(*input->rev == tree->first); - if (shallow) tree->second.info.revCount.reset(); - return {std::move(tree->second), input}; - } - // FIXME: check whether rev is an ancestor of ref. printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { @@ -395,21 +375,31 @@ struct GitInput : Input auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); - auto tree = Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = - !shallow - ? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))) - : std::nullopt, - .lastModified = lastModified - } - }; + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); - cacheGitInfo(*store, name, tree, *input->rev); + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); - return {std::move(tree), input}; + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); } }; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 3a767ef17..5cd43a74e 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -174,7 +174,7 @@ struct MercurialInput : Input auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) -> std::pair> { - input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1); + assert(input->rev); assert(!rev || rev == input->rev); return { Tree{ @@ -203,8 +203,13 @@ struct MercurialInput : Input {"ref", *input->ref}, }); - if (auto res = getCache()->lookup(store, mutableAttrs)) - return makeResult(res->first, std::move(res->second)); + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index bca581438..9faa5d9f6 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -11,7 +11,7 @@ repo=$TEST_ROOT/git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git* $TEST_ROOT/worktree $TEST_ROOT/shallow +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow git init $repo git -C $repo config user.email "foobar@example.com" @@ -59,10 +59,10 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath [[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] # Fetching with a explicit hash should succeed. -path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] -path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") [[ $(cat $path2/hello) = utrecht ]] mv ${repo}-tmp $repo @@ -99,7 +99,7 @@ path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\" # Committing should not affect the store path. git -C $repo commit -m 'Bla3' -a -path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchGit file://$repo).outPath") +path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] # tarball-ttl should be ignored if we specify a rev @@ -137,11 +137,10 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Nuke the cache -rm -rf $TEST_HOME/.cache/nix/gitv2 +rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH +# Try again, but without 'git' on PATH. This should fail. NIX=$(command -v nix) -# This should fail (! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) # Try again, with 'git' available. This should work.