From 0cbda84f5b14aba0416cb65f88f8e9d487895207 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:06:27 +0200 Subject: [PATCH] exportGit: Don't clone local repositories This ensures that commands like 'nix flake info /my/nixpkgs' don't copy a gigabyte of crap to ~/.cache/nix. Fixes #60. --- src/libexpr/primops/fetchGit.cc | 129 +++++++++++++++--------- src/libexpr/primops/fetchGit.hh | 2 +- src/libstore/http-binary-cache-store.cc | 5 +- tests/binary-cache.sh | 4 +- tests/fetchGit.sh | 2 + 5 files changed, 88 insertions(+), 54 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 3a6830cb7..eb95208de 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -18,14 +18,19 @@ namespace nix { extern std::regex revRegex; -GitInfo exportGit(ref store, const std::string & uri, +GitInfo exportGit(ref store, std::string uri, std::optional ref, std::optional rev, const std::string & name) { assert(!rev || rev->type == htSHA1); - if (!ref && !rev && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git"); + + // If this is a local directory (but not a file:// URI) and no ref + // or revision is given, then allow the use of an unclean working + // tree. + if (!ref && !rev && isLocal) { bool clean = true; @@ -66,67 +71,92 @@ GitInfo exportGit(ref store, const std::string & uri, return gitInfo; } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })), htSHA1); } - if (!ref) ref = "HEAD"s; + if (!ref) ref = isLocal ? "HEAD" : "master"; + + // Don't clone file:// URIs (but otherwise treat them the same as + // remote URIs, i.e. don't use the working tree or HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + if (!forceHttp && hasPrefix(uri, "file://")) { + uri = std::string(uri, 7); + isLocal = true; + } deletePath(getCacheDir() + "/nix/git"); Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + Path repoDir; - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } + if (isLocal) { - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + if (!rev) + rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1); + + if (!pathExists(cacheDir)) + createDirs(cacheDir); + + repoDir = uri; - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev) { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + repoDir = cacheDir; - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } - utimes(localRefFile.c_str(), times); + Path localRefFile = repoDir + "/refs/heads/" + *ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + st.st_mtime + settings.tarballTtl <= now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!rev) + rev = Hash(chomp(readFile(localRefFile)), htSHA1); } // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; gitInfo.ref = *ref; - gitInfo.rev = rev ? *rev : Hash(chomp(readFile(localRefFile)), htSHA1); + gitInfo.rev = *rev; printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); @@ -140,9 +170,10 @@ GitInfo exportGit(ref store, const std::string & uri, assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); - gitInfo.storePath = json["storePath"]; + Path storePath = json["storePath"]; - if (store->isValidPath(gitInfo.storePath)) { + if (store->isValidPath(storePath)) { + gitInfo.storePath = storePath; gitInfo.revCount = json["revCount"]; return gitInfo; } @@ -153,7 +184,7 @@ GitInfo exportGit(ref store, const std::string & uri, // FIXME: should pipe this, or find some better way to extract a // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev.gitRev() }); + auto tar = runProgram("git", true, { "-C", repoDir, "archive", gitInfo.rev.gitRev() }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -162,7 +193,7 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.storePath = store->addToStore(name, tmpDir); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev.gitRev() })); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index a867f38f6..32e748f98 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -14,7 +14,7 @@ struct GitInfo std::optional revCount; }; -GitInfo exportGit(ref store, const std::string & uri, +GitInfo exportGit(ref store, std::string uri, std::optional ref, std::optional rev, const std::string & name); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..105e1dcdd 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -160,10 +160,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..a3c3c7847 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -48,7 +48,7 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests @@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 51fd49e9f..d87ce8560 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,6 +9,8 @@ clearStore repo=$TEST_ROOT/git +export _NIX_FORCE_HTTP=1 + rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 git init $repo