lix-project/lix
19
42
Fork 18
Code Issues 295 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Git: Use unified caching system

Browse source
This commit is contained in:
Eelco Dolstra 2020-03-17 22:32:26 +01:00
parent d1165d8791
commit 38e360154d
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
3 changed files with 91 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
src/libstore/fetchers/git.cc
View file

@ -1,5 +1,6 @@
#include "fetchers.hh"
#include "parse.hh"
#include "fetchers/fetchers.hh"
#include "fetchers/cache.hh"
#include "fetchers/parse.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
@ -7,77 +8,15 @@
#include <sys/time.h>
#include <nlohmann/json.hpp>
using namespace std::string_literals;
namespace nix::fetchers {
static Path getCacheInfoPathFor(const std::string & name, const Hash & rev)
{
Path cacheDir = getCacheDir() + "/nix/git-revs-v2";
std::string linkName =
name == "source"
? rev.gitRev()
: hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false);
return cacheDir + "/" + linkName + ".link";
}
static std::string readHead(const Path & path)
{
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
}
static void cacheGitInfo(
Store & store,
const std::string & name,
const Tree & tree,
const Hash & rev)
{
if (!tree.info.revCount || !tree.info.lastModified) return;
nlohmann::json json;
json["storePath"] = store.printStorePath(tree.storePath);
json["name"] = name;
json["rev"] = rev.gitRev();
json["revCount"] = *tree.info.revCount;
json["lastModified"] = *tree.info.lastModified;
auto cacheInfoPath = getCacheInfoPathFor(name, rev);
createDirs(dirOf(cacheInfoPath));
writeFile(cacheInfoPath, json.dump());
}
static std::optional<std::pair<Hash, Tree>> lookupGitInfo(
ref<Store> store,
const std::string & name,
const Hash & rev)
{
try {
auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev)));
assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev);
auto storePath = store->parseStorePath((std::string) json["storePath"]);
if (store->isValidPath(storePath)) {
return {{rev, Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = json["revCount"],
.lastModified = json["lastModified"],
}
}}};
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
return {};
}
struct GitInput : Input
{
ParsedURL url;
@ -207,11 +146,38 @@ struct GitInput : Input
assert(!rev || rev->type == htSHA1);
auto cacheType = shallow ? "git-shallow" : "git";
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input
};
};
if (rev) {
if (auto tree = lookupGitInfo(store, name, *rev)) {
input->rev = tree->first;
return {std::move(tree->second), input};
}
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl();
@ -290,6 +256,13 @@ struct GitInput : Input
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
Path repoDir;
if (isLocal) {
@ -301,6 +274,14 @@ struct GitInput : Input
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
@ -368,16 +349,15 @@ struct GitInput : Input
if (isShallow && !shallow)
throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
if (auto tree = lookupGitInfo(store, name, *input->rev)) {
assert(*input->rev == tree->first);
if (shallow) tree->second.info.revCount.reset();
return {std::move(tree->second), input};
}
// FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
@ -395,21 +375,31 @@ struct GitInput : Input
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
auto tree = Tree {
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount =
!shallow
? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })))
: std::nullopt,
.lastModified = lastModified
}
};
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"lastModified", lastModified},
});
cacheGitInfo(*store, name, tree, *input->rev);
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
return {std::move(tree), input};
if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
getImmutableAttrs(),
infoAttrs,
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
}
};

11
src/libstore/fetchers/mercurial.cc
View file

@ -174,7 +174,7 @@ struct MercurialInput : Input
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1);
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree{
@ -203,8 +203,13 @@ struct MercurialInput : Input
{"ref", *input->ref},
});
if (auto res = getCache()->lookup(store, mutableAttrs))
return makeResult(res->first, std::move(res->second));
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));

13
tests/fetchGit.sh
View file

@ -11,7 +11,7 @@ repo=$TEST_ROOT/git
export _NIX_FORCE_HTTP=1
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git* $TEST_ROOT/worktree $TEST_ROOT/shallow
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow
git init $repo
git -C $repo config user.email "foobar@example.com"
@ -59,10 +59,10 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath
[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
# Fetching with a explicit hash should succeed.
path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
[[ $(cat $path2/hello) = utrecht ]]
mv ${repo}-tmp $repo
@ -99,7 +99,7 @@ path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"
# Committing should not affect the store path.
git -C $repo commit -m 'Bla3' -a
path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchGit file://$repo).outPath")
path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path2 = $path4 ]]
# tarball-ttl should be ignored if we specify a rev
@ -137,11 +137,10 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
# Nuke the cache
rm -rf $TEST_HOME/.cache/nix/gitv2
rm -rf $TEST_HOME/.cache/nix
# Try again, but without 'git' on PATH
# Try again, but without 'git' on PATH. This should fail.
NIX=$(command -v nix)
# This should fail
(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
# Try again, with 'git' available. This should work.