fetchMercurial: Don't fetch hashes we already have

This commit is contained in:
Eelco Dolstra 2017-11-01 18:43:11 +01:00
parent 1969f357b7
commit e026bc3b05
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 74 additions and 28 deletions

View file

@ -22,6 +22,8 @@ struct HgInfo
uint64_t revCount = 0; uint64_t revCount = 0;
}; };
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string & uri, HgInfo exportMercurial(ref<Store> store, const std::string & uri,
std::string rev, const std::string & name) std::string rev, const std::string & name)
{ {
@ -66,20 +68,28 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than tarball-ttl seconds, /* If we haven't pulled this repo less than tarball-ttl seconds,
do so now. FIXME: don't do this if "rev" is a hash and we do so now. */
fetched it previously */
time_t now = time(0); time_t now = time(0);
struct stat st; struct stat st;
if (stat(stampFile.c_str(), &st) != 0 || if (stat(stampFile.c_str(), &st) != 0 ||
st.st_mtime < now - settings.tarballTtl) st.st_mtime < now - settings.tarballTtl)
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); /* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex)
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) { if (pathExists(cacheDir)) {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
} else { } else {
createDirs(dirOf(cacheDir)); createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
}
} }
writeFile(stampFile, ""); writeFile(stampFile, "");

View file

@ -895,32 +895,46 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
string runProgram(Path program, bool searchPath, const Strings & args, string runProgram(Path program, bool searchPath, const Strings & args,
const std::experimental::optional<std::string> & input) const std::experimental::optional<std::string> & input)
{
RunOptions opts(program, args);
opts.searchPath = searchPath;
opts.input = input;
auto res = runProgram(opts);
if (!statusOk(res.first))
throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
return res.second;
}
std::pair<int, std::string> runProgram(const RunOptions & options)
{ {
checkInterrupt(); checkInterrupt();
/* Create a pipe. */ /* Create a pipe. */
Pipe out, in; Pipe out, in;
out.create(); out.create();
if (input) in.create(); if (options.input) in.create();
/* Fork. */ /* Fork. */
Pid pid = startProcess([&]() { Pid pid = startProcess([&]() {
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("dupping stdout"); throw SysError("dupping stdout");
if (input && dup2(in.readSide.get(), STDIN_FILENO) == -1) if (options.input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin"); throw SysError("dupping stdin");
Strings args_(args); Strings args_(options.args);
args_.push_front(program); args_.push_front(options.program);
restoreSignals(); restoreSignals();
if (searchPath) if (options.searchPath)
execvp(program.c_str(), stringsToCharPtrs(args_).data()); execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
else else
execv(program.c_str(), stringsToCharPtrs(args_).data()); execv(options.program.c_str(), stringsToCharPtrs(args_).data());
throw SysError(format("executing '%1%'") % program); throw SysError("executing '%1%'", options.program);
}); });
out.writeSide = -1; out.writeSide = -1;
@ -935,11 +949,11 @@ string runProgram(Path program, bool searchPath, const Strings & args,
}); });
if (input) { if (options.input) {
in.readSide = -1; in.readSide = -1;
writerThread = std::thread([&]() { writerThread = std::thread([&]() {
try { try {
writeFull(in.writeSide.get(), *input); writeFull(in.writeSide.get(), *options.input);
promise.set_value(); promise.set_value();
} catch (...) { } catch (...) {
promise.set_exception(std::current_exception()); promise.set_exception(std::current_exception());
@ -952,14 +966,11 @@ string runProgram(Path program, bool searchPath, const Strings & args,
/* Wait for the child to finish. */ /* Wait for the child to finish. */
int status = pid.wait(); int status = pid.wait();
if (!statusOk(status))
throw ExecError(status, format("program '%1%' %2%")
% program % statusToString(status));
/* Wait for the writer thread to finish. */ /* Wait for the writer thread to finish. */
if (input) promise.get_future().get(); if (options.input) promise.get_future().get();
return result; return {status, result};
} }

View file

@ -245,6 +245,23 @@ string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(), const Strings & args = Strings(),
const std::experimental::optional<std::string> & input = {}); const std::experimental::optional<std::string> & input = {});
struct RunOptions
{
Path program;
bool searchPath = true;
Strings args;
std::experimental::optional<std::string> input;
bool _killStderr = false;
RunOptions(const Path & program, const Strings & args)
: program(program), args(args) { };
RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
};
std::pair<int, std::string> runProgram(const RunOptions & options);
class ExecError : public Error class ExecError : public Error
{ {
public: public:

View file

@ -9,7 +9,7 @@ clearStore
repo=$TEST_ROOT/hg repo=$TEST_ROOT/hg
rm -rfv $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
hg init $repo hg init $repo
echo '[ui]' >> $repo/.hg/hgrc echo '[ui]' >> $repo/.hg/hgrc
@ -24,13 +24,14 @@ echo world > $repo/hello
hg commit --cwd $repo -m 'Bla2' hg commit --cwd $repo -m 'Bla2'
rev2=$(hg log --cwd $repo -r tip --template '{node}') rev2=$(hg log --cwd $repo -r tip --template '{node}')
hg log --cwd $repo # Fetch the default branch.
hg log --cwd $repo -r tip --template '{node}\n'
path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
[[ $(cat $path/hello) = world ]] [[ $(cat $path/hello) = world ]]
# Fetch using an explicit revision hash.
path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
# Fetch again. This should be cached. # Fetch again. This should be cached.
mv $repo ${repo}-tmp mv $repo ${repo}-tmp
path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
@ -43,6 +44,13 @@ path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
# But with TTL 0, it should fail. # But with TTL 0, it should fail.
(! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)") (! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)")
# Fetching with a explicit hash should succeed.
path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
[[ $(cat $path2/hello) = utrecht ]]
mv ${repo}-tmp $repo mv ${repo}-tmp $repo
# Using a clean working tree should produce the same result. # Using a clean working tree should produce the same result.