exportGit: Don't clone local repositories

This ensures that commands like 'nix flake info /my/nixpkgs' don't
copy a gigabyte of crap to ~/.cache/nix.

Fixes #60.
This commit is contained in:
Eelco Dolstra 2019-04-19 14:06:27 +02:00
parent 160ce18a0e
commit 0cbda84f5b
5 changed files with 88 additions and 54 deletions

View file

@ -18,14 +18,19 @@ namespace nix {
extern std::regex revRegex; extern std::regex revRegex;
GitInfo exportGit(ref<Store> store, const std::string & uri, GitInfo exportGit(ref<Store> store, std::string uri,
std::optional<std::string> ref, std::optional<std::string> ref,
std::optional<Hash> rev, std::optional<Hash> rev,
const std::string & name) const std::string & name)
{ {
assert(!rev || rev->type == htSHA1); assert(!rev || rev->type == htSHA1);
if (!ref && !rev && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git");
// If this is a local directory (but not a file:// URI) and no ref
// or revision is given, then allow the use of an unclean working
// tree.
if (!ref && !rev && isLocal) {
bool clean = true; bool clean = true;
@ -66,67 +71,92 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
return gitInfo; return gitInfo;
} }
// clean working tree, but no ref or rev specified. Use 'HEAD'.
rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })), htSHA1);
} }
if (!ref) ref = "HEAD"s; if (!ref) ref = isLocal ? "HEAD" : "master";
// Don't clone file:// URIs (but otherwise treat them the same as
// remote URIs, i.e. don't use the working tree or HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
if (!forceHttp && hasPrefix(uri, "file://")) {
uri = std::string(uri, 7);
isLocal = true;
}
deletePath(getCacheDir() + "/nix/git"); deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
Path repoDir;
if (!pathExists(cacheDir)) { if (isLocal) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", cacheDir });
}
Path localRefFile = cacheDir + "/refs/heads/" + *ref; if (!rev)
rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1);
if (!pathExists(cacheDir))
createDirs(cacheDir);
repoDir = uri;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev) {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else { } else {
/* If the local ref is older than tarball-ttl seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
st.st_mtime + settings.tarballTtl <= now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so repoDir = cacheDir;
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
struct timeval times[2]; if (!pathExists(cacheDir)) {
times[0].tv_sec = now; createDirs(dirOf(cacheDir));
times[0].tv_usec = 0; runProgram("git", true, { "init", "--bare", repoDir });
times[1].tv_sec = now; }
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times); Path localRefFile = repoDir + "/refs/heads/" + *ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than tarball-ttl seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
st.st_mtime + settings.tarballTtl <= now;
}
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
if (!rev)
rev = Hash(chomp(readFile(localRefFile)), htSHA1);
} }
// FIXME: check whether rev is an ancestor of ref. // FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo; GitInfo gitInfo;
gitInfo.ref = *ref; gitInfo.ref = *ref;
gitInfo.rev = rev ? *rev : Hash(chomp(readFile(localRefFile)), htSHA1); gitInfo.rev = *rev;
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
@ -140,9 +170,10 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev);
gitInfo.storePath = json["storePath"]; Path storePath = json["storePath"];
if (store->isValidPath(gitInfo.storePath)) { if (store->isValidPath(storePath)) {
gitInfo.storePath = storePath;
gitInfo.revCount = json["revCount"]; gitInfo.revCount = json["revCount"];
return gitInfo; return gitInfo;
} }
@ -153,7 +184,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
// FIXME: should pipe this, or find some better way to extract a // FIXME: should pipe this, or find some better way to extract a
// revision. // revision.
auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev.gitRev() }); auto tar = runProgram("git", true, { "-C", repoDir, "archive", gitInfo.rev.gitRev() });
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true); AutoDelete delTmpDir(tmpDir, true);
@ -162,7 +193,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev.gitRev() })); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() }));
nlohmann::json json; nlohmann::json json;
json["storePath"] = gitInfo.storePath; json["storePath"] = gitInfo.storePath;

View file

@ -14,7 +14,7 @@ struct GitInfo
std::optional<uint64_t> revCount; std::optional<uint64_t> revCount;
}; };
GitInfo exportGit(ref<Store> store, const std::string & uri, GitInfo exportGit(ref<Store> store, std::string uri,
std::optional<std::string> ref, std::optional<std::string> ref,
std::optional<Hash> rev, std::optional<Hash> rev,
const std::string & name); const std::string & name);

View file

@ -160,10 +160,11 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params) const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store>
{ {
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
if (std::string(uri, 0, 7) != "http://" && if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" && std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") (!forceHttp || std::string(uri, 0, 7) != "file://"))
) return 0; return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init(); store->init();
return store; return store;

View file

@ -48,7 +48,7 @@ basicTests
# Test HttpBinaryCacheStore. # Test HttpBinaryCacheStore.
export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 export _NIX_FORCE_HTTP=1
basicTests basicTests
@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
otherKey="$(cat $TEST_ROOT/pk3)" otherKey="$(cat $TEST_ROOT/pk3)"
_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
# Downloading should fail if we don't provide a key. # Downloading should fail if we don't provide a key.

View file

@ -9,6 +9,8 @@ clearStore
repo=$TEST_ROOT/git repo=$TEST_ROOT/git
export _NIX_FORCE_HTTP=1
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
git init $repo git init $repo