From a8fe0dc16c18c260a1bb9e88f467e4c929cf22dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 15:50:35 +0100 Subject: [PATCH] Speed up fetching submodules Previously we would completely refetch the submodules from the network, even though the repo might already have them. Now we copy the .git/modules directory from the repo as an optimisation. This speeds up evaluating builtins.fetchTree { type = "git"; url = "/path/to/blender"; submodules = true; } (where /path/to/blender already has the needed submodules) from 121s to 57s. This is still pretty inefficient and a hack, but a better solution is best done on the lazy-trees branch. This change also help in the case where the repo already has the submodules but the origin is unfetchable for whatever reason (e.g. there have been cases where Nix in a GitHub action doesn't have the right authentication set up). --- src/libfetchers/git.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8981476e1..9908db214 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -639,6 +639,14 @@ struct GitInputScheme : InputScheme } else runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });