From f58604ac32d0ab0718a0e68b3c1b050e4ab121cb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Fri, 25 Oct 2019 17:45:14 +0200 Subject: [PATCH 01/17] Add fetchSubmodules to builtins.fetchGit There are some downsides to this features: - Submodules are not cached (unlike the root repo), - Full checkouts are created in a temporary directory. --- src/libexpr/primops/fetchGit.cc | 45 +++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..b08e48404 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,13 +22,14 @@ struct GitInfo std::string rev; std::string shortRev; uint64_t revCount = 0; + bool submodules = false; }; std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, - const std::string & name) + const std::string & name, bool fetchSubmodules) { if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -145,7 +146,8 @@ GitInfo exportGit(ref store, const std::string & uri, printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev + + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken @@ -165,16 +167,35 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - unpackTarfile(*source, tmpDir); + // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root + // repo is cached. + if (fetchSubmodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + + deletePath(tmpDir + "/.git"); + + gitInfo.submodules = true; + } else { + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); @@ -198,6 +219,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va std::optional ref; std::string rev; std::string name = "source"; + bool fetchSubmodules = false; PathSet context; state.forceValue(*args[0]); @@ -216,6 +238,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "fetchSubmodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); } @@ -230,13 +254,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules); v.attrs->sort(); if (state.allowedPaths) From c2a24c2b8876a0862ede39f63e684cb875cc7aab Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:09:50 +0200 Subject: [PATCH 02/17] Add test for fetchGit submodule support --- tests/fetchGitSubmodules.sh | 50 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 51 insertions(+) create mode 100644 tests/fetchGitSubmodules.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh new file mode 100644 index 000000000..c821d2caa --- /dev/null +++ b/tests/fetchGitSubmodules.sh @@ -0,0 +1,50 @@ +source common.sh + +set -u + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git submodule tests" + exit 99 +fi + +clearStore + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2 + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + + diff --git a/tests/local.mk b/tests/local.mk index dab3a23b6..01fac4fcd 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ + fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ run.sh \ From ea861be292d790db9835c08abee734baa77b9cf7 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:13:08 +0200 Subject: [PATCH 03/17] Add documentation for submodule support in fetchGit --- doc/manual/expressions/builtins.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 394e1fc32..838edae3c 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -422,6 +422,16 @@ stdenv.mkDerivation { … } + + fetchSubmodules + + + A boolean parameter that specifies whether submodules + should be checked out. Defaults to + false. + + + From c8d33de77781fe7850d04a7374c6c22a2e995b70 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:42:19 +0100 Subject: [PATCH 04/17] Add git submodule fixes from @bjornfor This fixes fetching repositories with no submodules and also cleans up .git files in checkouts. --- src/libexpr/local.mk | 1 + src/libexpr/primops/fetchGit.cc | 9 +++++++-- tests/fetchGitSubmodules.sh | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = + ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b08e48404..c58d28bbc 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "tarfile.hh" +#include #include #include @@ -182,9 +183,13 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - deletePath(tmpDir + "/.git"); + for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { + if (p.path().filename() == ".git") { + std::filesystem::remove_all(p.path()); + } + } gitInfo.submodules = true; } else { diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c821d2caa..987bfd69e 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -47,4 +47,5 @@ pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] - +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" From c846abb5cc900d5fcb5aeb48427a1eac646cc0f3 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:46:19 +0100 Subject: [PATCH 05/17] Add more test for git submodule functionality --- tests/fetchGitSubmodules.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 987bfd69e..c252d42f7 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -40,12 +40,23 @@ rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 435366ed3c292756bb74fd12f47ff96e93cd48ac Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:50:13 +0100 Subject: [PATCH 06/17] Rename fetchGit fetchSubmodules to just submodules --- src/libexpr/primops/fetchGit.cc | 2 +- tests/fetchGitSubmodules.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index c58d28bbc..e8905b548 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -243,7 +243,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "fetchSubmodules") + else if (n == "submodules") fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c252d42f7..6e4ef270b 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -39,8 +39,8 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] @@ -57,6 +57,6 @@ test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 6c00a9545f2162496032c490aadbadd85856662c Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 15:50:39 +0100 Subject: [PATCH 07/17] Fix typo in submodule test --- tests/fetchGitSubmodules.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 6e4ef270b..9a21421cc 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -56,7 +56,8 @@ pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$roo test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. -noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From cc4fe977e5eedf00d8e3d267ccbc3676b3fac1a0 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 9 Mar 2020 14:07:26 +0100 Subject: [PATCH 08/17] Link to stdc++fs Some platforms seem to still require linking with stdc++fs to enable STL std::filesystem support. --- src/libexpr/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9b5fcc561..b6ee82424 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,11 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust +ifeq ($(CXX), g++) +libexpr_LDFLAGS = -lstdc++fs +else libexpr_LDFLAGS = +endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl From 002a3a95dcb7391a8372ba7a74f4e7e9bd48b59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:21 +0100 Subject: [PATCH 09/17] fetchGit: fix "fatal: couldn't find remote ref refs/heads/master" issue with submodules --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e8905b548..66259d778 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -179,7 +179,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + "--", cacheDir, fmt("%s", *ref) }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 587e259bfdda248fb2d81389faca816b8efdb5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:55 +0100 Subject: [PATCH 10/17] tests/fetchGitSubmodules: add more tests --- tests/fetchGitSubmodules.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 9a21421cc..8ee1bb067 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -42,6 +42,25 @@ pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootR pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 6864ad7cf510d96f0cfc0cdbeffc1188a4eab44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:14 +0100 Subject: [PATCH 11/17] fetchGit: fix submodule output attribute Before this change it would be false for all evaluations but the first. Now it follows the input argument (as it should). --- src/libexpr/primops/fetchGit.cc | 7 +++---- tests/fetchGitSubmodules.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66259d778..653d99b53 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -32,6 +32,9 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) { + GitInfo gitInfo; + gitInfo.submodules = fetchSubmodules; + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -49,7 +52,6 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -141,7 +143,6 @@ GitInfo exportGit(ref store, const std::string & uri, } // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -190,8 +191,6 @@ GitInfo exportGit(ref store, const std::string & uri, std::filesystem::remove_all(p.path()); } } - - gitInfo.submodules = true; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 8ee1bb067..2d625c376 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -61,6 +61,15 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo [[ $r6 == $r7 ]] [[ $r7 == $r8 ]] +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 369fffd6f1f7d939e528b3591dca49989c51ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:58 +0100 Subject: [PATCH 12/17] fetchGit: add submodules attribute to the .link file The .link file is used as a lock, so I think we should put the "submodule" attribute in there since turning on submodules creates a new .link file path. --- src/libexpr/primops/fetchGit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 653d99b53..969b1a4ed 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -211,6 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, json["name"] = name; json["rev"] = gitInfo.rev; json["revCount"] = gitInfo.revCount; + json["submodules"] = gitInfo.submodules; writeFile(storeLink, json.dump()); From be84049baf047fd2eee08def5c8013797c0832a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 20:14:19 +0100 Subject: [PATCH 13/17] tests/fetchGitSubmodules.sh: more checks --- tests/fetchGitSubmodules.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 2d625c376..30a8737bd 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -38,10 +38,6 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) -pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") - r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") @@ -70,16 +66,26 @@ have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\" have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] # Checking out the same repo with submodules returns in the same store path. [[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + # The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" From b306b7039ef3f2b76dbcc8020dcf67ed27442e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:44:37 +0100 Subject: [PATCH 14/17] fetchGit: checkout rev instead of latest ref Major bugfix for the submodules = true code path. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 969b1a4ed..0c6bc6dc0 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -182,7 +182,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--", cacheDir, fmt("%s", *ref) }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); From cc522d0d23b71faeac95998d264abe14ddceeed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:45:31 +0100 Subject: [PATCH 15/17] fetchGit: fix submodules = true for dirty trees --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 0c6bc6dc0..66e1d7b98 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -55,8 +55,12 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" }); + if (fetchSubmodules) { + gitOpts.emplace_back("--recurse-submodules"); + } auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); + runProgram("git", true, gitOpts), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, uri)); From f686efeed4d6305101c56136855e2d6b87e649e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Fri, 20 Mar 2020 14:36:10 +0100 Subject: [PATCH 16/17] fetchGit: fix submodule corner case by fetching all refs from cacheDir Due to fetchGit not checking if rev is an ancestor of ref (there is even a FIXME comment about it in the code), the cache repo might not have the ref even though it has the rev. This doesn't matter when submodule = false, but the submodule = true code blows up because it tries to fetch the (missing) ref from the cache repo. Fix this in the simplest way possible: fetch all refs from the local cache repo when submodules = true. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66e1d7b98..f138b755c 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -183,8 +183,11 @@ GitInfo exportGit(ref store, const std::string & uri, AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: the cacheDir repo might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch everything to + // ensure we get the rev. runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s", *ref) }); + "--update-head-ok", "--", cacheDir, "refs/*:refs/*" }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 40c023ecfe49fea6e66db34c5f841fcf7001cbeb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sun, 29 Mar 2020 23:47:48 +0200 Subject: [PATCH 17/17] fetchGit: don't use std::filesystem to filter git repos Using std::filesystem means also having to link with -lstdc++fs on some platforms and it's hard to discover for what platforms this is needed. As all the functionality is already implemented as utilities, use those instead. --- src/libexpr/local.mk | 4 ---- src/libexpr/primops/fetchGit.cc | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b6ee82424..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,11 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust -ifeq ($(CXX), g++) -libexpr_LDFLAGS = -lstdc++fs -else libexpr_LDFLAGS = -endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f138b755c..d60e6b803 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,7 +6,6 @@ #include "hash.hh" #include "tarfile.hh" -#include #include #include @@ -28,6 +27,13 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) @@ -175,6 +181,7 @@ GitInfo exportGit(ref store, const std::string & uri, Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root // repo is cached. @@ -193,11 +200,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { - if (p.path().filename() == ".git") { - std::filesystem::remove_all(p.path()); - } - } + filter = isNotDotGitDirectory; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); @@ -208,7 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, unpackTarfile(*source, tmpDir); } - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); + gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter)); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));