Merge branch 'fetchgit-recursive' of https://github.com/blitz/nix
This commit is contained in:
commit
55cefd41d6
|
@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
|
||||||
</para>
|
</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
<varlistentry>
|
||||||
|
<term>submodules</term>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
A Boolean parameter that specifies whether submodules
|
||||||
|
should be checked out. Defaults to
|
||||||
|
<literal>false</literal>.
|
||||||
|
</para>
|
||||||
|
</listitem>
|
||||||
|
</varlistentry>
|
||||||
</variablelist>
|
</variablelist>
|
||||||
|
|
||||||
<example>
|
<example>
|
||||||
|
|
|
@ -13,6 +13,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
std::optional<std::string> ref;
|
std::optional<std::string> ref;
|
||||||
std::optional<Hash> rev;
|
std::optional<Hash> rev;
|
||||||
std::string name = "source";
|
std::string name = "source";
|
||||||
|
bool fetchSubmodules = false;
|
||||||
PathSet context;
|
PathSet context;
|
||||||
|
|
||||||
state.forceValue(*args[0]);
|
state.forceValue(*args[0]);
|
||||||
|
@ -31,6 +32,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
|
rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
|
||||||
else if (n == "name")
|
else if (n == "name")
|
||||||
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||||
|
else if (n == "submodules")
|
||||||
|
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
|
||||||
else
|
else
|
||||||
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
|
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
|
||||||
}
|
}
|
||||||
|
@ -48,15 +51,15 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
if (evalSettings.pureEval && !rev)
|
if (evalSettings.pureEval && !rev)
|
||||||
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
|
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
|
||||||
|
|
||||||
auto parsedUrl = parseURL(
|
fetchers::Attrs attrs;
|
||||||
url.find("://") != std::string::npos
|
attrs.insert_or_assign("type", "git");
|
||||||
? "git+" + url
|
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
|
||||||
: "git+file://" + url);
|
if (ref) attrs.insert_or_assign("ref", *ref);
|
||||||
if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
|
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
|
||||||
if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
|
if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
|
||||||
// FIXME: use name
|
auto input = fetchers::inputFromAttrs(attrs);
|
||||||
auto input = fetchers::inputFromURL(parsedUrl);
|
|
||||||
|
|
||||||
|
// FIXME: use name?
|
||||||
auto [tree, input2] = input->fetchTree(state.store);
|
auto [tree, input2] = input->fetchTree(state.store);
|
||||||
|
|
||||||
state.mkAttrs(v, 8);
|
state.mkAttrs(v, 8);
|
||||||
|
@ -70,6 +73,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
|
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
|
||||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
|
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
|
||||||
tree.info.revCount.value_or(0));
|
tree.info.revCount.value_or(0));
|
||||||
|
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
|
||||||
v.attrs->sort();
|
v.attrs->sort();
|
||||||
|
|
||||||
if (state.allowedPaths)
|
if (state.allowedPaths)
|
||||||
|
|
|
@ -15,12 +15,20 @@ static std::string readHead(const Path & path)
|
||||||
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
|
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool isNotDotGitDirectory(const Path & path)
|
||||||
|
{
|
||||||
|
static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
|
||||||
|
|
||||||
|
return not std::regex_match(path, gitDirRegex);
|
||||||
|
}
|
||||||
|
|
||||||
struct GitInput : Input
|
struct GitInput : Input
|
||||||
{
|
{
|
||||||
ParsedURL url;
|
ParsedURL url;
|
||||||
std::optional<std::string> ref;
|
std::optional<std::string> ref;
|
||||||
std::optional<Hash> rev;
|
std::optional<Hash> rev;
|
||||||
bool shallow = false;
|
bool shallow = false;
|
||||||
|
bool submodules = false;
|
||||||
|
|
||||||
GitInput(const ParsedURL & url) : url(url)
|
GitInput(const ParsedURL & url) : url(url)
|
||||||
{ }
|
{ }
|
||||||
|
@ -66,6 +74,8 @@ struct GitInput : Input
|
||||||
attrs.emplace("rev", rev->gitRev());
|
attrs.emplace("rev", rev->gitRev());
|
||||||
if (shallow)
|
if (shallow)
|
||||||
attrs.emplace("shallow", true);
|
attrs.emplace("shallow", true);
|
||||||
|
if (submodules)
|
||||||
|
attrs.emplace("submodules", true);
|
||||||
return attrs;
|
return attrs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,7 +97,9 @@ struct GitInput : Input
|
||||||
|
|
||||||
assert(!rev || rev->type == htSHA1);
|
assert(!rev || rev->type == htSHA1);
|
||||||
|
|
||||||
auto cacheType = shallow ? "git-shallow" : "git";
|
std::string cacheType = "git";
|
||||||
|
if (shallow) cacheType += "-shallow";
|
||||||
|
if (submodules) cacheType += "-submodules";
|
||||||
|
|
||||||
auto getImmutableAttrs = [&]()
|
auto getImmutableAttrs = [&]()
|
||||||
{
|
{
|
||||||
|
@ -161,8 +173,12 @@ struct GitInput : Input
|
||||||
if (settings.warnDirty)
|
if (settings.warnDirty)
|
||||||
warn("Git tree '%s' is dirty", actualUrl);
|
warn("Git tree '%s' is dirty", actualUrl);
|
||||||
|
|
||||||
|
auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
|
||||||
|
if (submodules)
|
||||||
|
gitOpts.emplace_back("--recurse-submodules");
|
||||||
|
|
||||||
auto files = tokenizeString<std::set<std::string>>(
|
auto files = tokenizeString<std::set<std::string>>(
|
||||||
runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s);
|
runProgram("git", true, gitOpts), "\0"s);
|
||||||
|
|
||||||
PathFilter filter = [&](const Path & p) -> bool {
|
PathFilter filter = [&](const Path & p) -> bool {
|
||||||
assert(hasPrefix(p, actualUrl));
|
assert(hasPrefix(p, actualUrl));
|
||||||
|
@ -299,20 +315,39 @@ struct GitInput : Input
|
||||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||||
return makeResult(res->first, std::move(res->second));
|
return makeResult(res->first, std::move(res->second));
|
||||||
|
|
||||||
// FIXME: should pipe this, or find some better way to extract a
|
|
||||||
// revision.
|
|
||||||
auto source = sinkToSource([&](Sink & sink) {
|
|
||||||
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
|
|
||||||
gitOptions.standardOut = &sink;
|
|
||||||
runProgram2(gitOptions);
|
|
||||||
});
|
|
||||||
|
|
||||||
Path tmpDir = createTempDir();
|
Path tmpDir = createTempDir();
|
||||||
AutoDelete delTmpDir(tmpDir, true);
|
AutoDelete delTmpDir(tmpDir, true);
|
||||||
|
PathFilter filter = defaultPathFilter;
|
||||||
|
|
||||||
unpackTarfile(*source, tmpDir);
|
if (submodules) {
|
||||||
|
Path tmpGitDir = createTempDir();
|
||||||
|
AutoDelete delTmpGitDir(tmpGitDir, true);
|
||||||
|
|
||||||
auto storePath = store->addToStore(name, tmpDir);
|
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
|
||||||
|
// TODO: repoDir might lack the ref (it only checks if rev
|
||||||
|
// exists, see FIXME above) so use a big hammer and fetch
|
||||||
|
// everything to ensure we get the rev.
|
||||||
|
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
|
||||||
|
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
|
||||||
|
|
||||||
|
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
|
||||||
|
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
|
||||||
|
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
|
||||||
|
|
||||||
|
filter = isNotDotGitDirectory;
|
||||||
|
} else {
|
||||||
|
// FIXME: should pipe this, or find some better way to extract a
|
||||||
|
// revision.
|
||||||
|
auto source = sinkToSource([&](Sink & sink) {
|
||||||
|
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
|
||||||
|
gitOptions.standardOut = &sink;
|
||||||
|
runProgram2(gitOptions);
|
||||||
|
});
|
||||||
|
|
||||||
|
unpackTarfile(*source, tmpDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
|
||||||
|
|
||||||
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
|
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
|
||||||
|
|
||||||
|
@ -378,7 +413,7 @@ struct GitInputScheme : InputScheme
|
||||||
if (maybeGetStrAttr(attrs, "type") != "git") return {};
|
if (maybeGetStrAttr(attrs, "type") != "git") return {};
|
||||||
|
|
||||||
for (auto & [name, value] : attrs)
|
for (auto & [name, value] : attrs)
|
||||||
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow")
|
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
|
||||||
throw Error("unsupported Git input attribute '%s'", name);
|
throw Error("unsupported Git input attribute '%s'", name);
|
||||||
|
|
||||||
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
|
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
|
||||||
|
@ -392,6 +427,8 @@ struct GitInputScheme : InputScheme
|
||||||
|
|
||||||
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
|
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
|
||||||
|
|
||||||
|
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
|
||||||
|
|
||||||
return input;
|
return input;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
97
tests/fetchGitSubmodules.sh
Normal file
97
tests/fetchGitSubmodules.sh
Normal file
|
@ -0,0 +1,97 @@
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
if [[ -z $(type -p git) ]]; then
|
||||||
|
echo "Git not installed; skipping Git submodule tests"
|
||||||
|
exit 99
|
||||||
|
fi
|
||||||
|
|
||||||
|
clearStore
|
||||||
|
|
||||||
|
rootRepo=$TEST_ROOT/gitSubmodulesRoot
|
||||||
|
subRepo=$TEST_ROOT/gitSubmodulesSub
|
||||||
|
|
||||||
|
rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix
|
||||||
|
|
||||||
|
initGitRepo() {
|
||||||
|
git init $1
|
||||||
|
git -C $1 config user.email "foobar@example.com"
|
||||||
|
git -C $1 config user.name "Foobar"
|
||||||
|
}
|
||||||
|
|
||||||
|
addGitContent() {
|
||||||
|
echo "lorem ipsum" > $1/content
|
||||||
|
git -C $1 add content
|
||||||
|
git -C $1 commit -m "Initial commit"
|
||||||
|
}
|
||||||
|
|
||||||
|
initGitRepo $subRepo
|
||||||
|
addGitContent $subRepo
|
||||||
|
|
||||||
|
initGitRepo $rootRepo
|
||||||
|
|
||||||
|
git -C $rootRepo submodule init
|
||||||
|
git -C $rootRepo submodule add $subRepo sub
|
||||||
|
git -C $rootRepo add sub
|
||||||
|
git -C $rootRepo commit -m "Add submodule"
|
||||||
|
|
||||||
|
rev=$(git -C $rootRepo rev-parse HEAD)
|
||||||
|
|
||||||
|
r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
|
||||||
|
r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
|
||||||
|
r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
|
||||||
|
[[ $r1 == $r2 ]]
|
||||||
|
[[ $r2 != $r3 ]]
|
||||||
|
|
||||||
|
r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath")
|
||||||
|
r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath")
|
||||||
|
r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
|
||||||
|
[[ $r1 == $r4 ]]
|
||||||
|
[[ $r4 == $r5 ]]
|
||||||
|
[[ $r3 == $r6 ]]
|
||||||
|
[[ $r6 == $r7 ]]
|
||||||
|
[[ $r7 == $r8 ]]
|
||||||
|
|
||||||
|
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
|
||||||
|
[[ $have_submodules == false ]]
|
||||||
|
|
||||||
|
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
|
||||||
|
[[ $have_submodules == false ]]
|
||||||
|
|
||||||
|
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
|
||||||
|
[[ $have_submodules == true ]]
|
||||||
|
|
||||||
|
pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
|
||||||
|
pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
|
||||||
|
|
||||||
|
# The resulting store path cannot be the same.
|
||||||
|
[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
|
||||||
|
|
||||||
|
# Checking out the same repo with submodules returns in the same store path.
|
||||||
|
[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
|
||||||
|
|
||||||
|
# Checking out the same repo with submodules returns in the same store path.
|
||||||
|
[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]
|
||||||
|
|
||||||
|
# The submodules flag is actually honored.
|
||||||
|
[[ ! -e $pathWithoutSubmodules/sub/content ]]
|
||||||
|
[[ -e $pathWithSubmodules/sub/content ]]
|
||||||
|
|
||||||
|
[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]
|
||||||
|
|
||||||
|
# No .git directory or submodule reference files must be left
|
||||||
|
test "$(find "$pathWithSubmodules" -name .git)" = ""
|
||||||
|
|
||||||
|
# Git repos without submodules can be fetched with submodules = true.
|
||||||
|
subRev=$(git -C $subRepo rev-parse HEAD)
|
||||||
|
noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
|
||||||
|
noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
|
||||||
|
|
||||||
|
[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]
|
|
@ -17,6 +17,7 @@ nix_tests = \
|
||||||
nar-access.sh \
|
nar-access.sh \
|
||||||
structured-attrs.sh \
|
structured-attrs.sh \
|
||||||
fetchGit.sh \
|
fetchGit.sh \
|
||||||
|
fetchGitSubmodules.sh \
|
||||||
fetchMercurial.sh \
|
fetchMercurial.sh \
|
||||||
signing.sh \
|
signing.sh \
|
||||||
run.sh \
|
run.sh \
|
||||||
|
|
Loading…
Reference in a new issue