From 1969f357b7f9abbff82b613b88ae53b84ff4e483 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2017 16:32:53 +0100 Subject: [PATCH] Add fetchMercurial primop E.g. $ nix eval '(fetchMercurial https://www.mercurial-scm.org/repo/hello)' { branch = "default"; outPath = "/nix/store/alvb9y1kfz42bjishqmyy3pphnrh1pfa-source"; rev = "82e55d328c8ca4ee16520036c0aaace03a5beb65"; revCount = 1; shortRev = "82e55d328c8c"; } $ nix eval '(fetchMercurial { url = https://www.mercurial-scm.org/repo/hello; rev = "0a04b987be5ae354b710cefeba0e2d9de7ad41a9"; })' { branch = "default"; outPath = "/nix/store/alvb9y1kfz42bjishqmyy3pphnrh1pfa-source"; rev = "0a04b987be5ae354b710cefeba0e2d9de7ad41a9"; revCount = 0; shortRev = "0a04b987be5a"; } $ nix eval '(fetchMercurial /tmp/unclean-hg-tree)' { branch = "default"; outPath = "/nix/store/cm750cdw1x8wfpm3jq7mz09r30l9r024-source"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "000000000000"; } --- release.nix | 2 +- shell.nix | 3 + src/libexpr/primops/fetchMercurial.cc | 188 ++++++++++++++++++++++++++ src/libexpr/primops/fetchgit.cc | 3 +- src/libstore/download.cc | 2 +- tests/fetchMercurial.sh | 73 ++++++++++ tests/local.mk | 3 +- 7 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 src/libexpr/primops/fetchMercurial.cc create mode 100644 tests/fetchMercurial.sh diff --git a/release.nix b/release.nix index a98199258..f76e7cb9e 100644 --- a/release.nix +++ b/release.nix @@ -76,7 +76,7 @@ let [ curl bzip2 xz brotli openssl pkgconfig sqlite boehmgc - + mercurial ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/shell.nix b/shell.nix index 2b4d0dd20..02fbd59b1 100644 --- a/shell.nix +++ b/shell.nix @@ -23,6 +23,9 @@ with import ./release-common.nix { inherit pkgs; }; # For nix-perl perl perlPackages.DBDSQLite + + # Tests + mercurial ] ++ lib.optional stdenv.isLinux libseccomp; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc new file mode 100644 index 000000000..391ce45a0 --- /dev/null +++ b/src/libexpr/primops/fetchMercurial.cc @@ -0,0 +1,188 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" +#include "pathlocks.hh" + +#include + +#include + +#include + +using namespace std::string_literals; + +namespace nix { + +struct HgInfo +{ + Path storePath; + std::string branch; + std::string rev; + uint64_t revCount = 0; +}; + +HgInfo exportMercurial(ref store, const std::string & uri, + std::string rev, const std::string & name) +{ + if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + printTalkative("copying unclean Mercurial working tree '%s'", uri); + + HgInfo hgInfo; + hgInfo.rev = "0000000000000000000000000000000000000000"; + hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, uri)); + auto st = lstat(p); + std::string file(p, uri.size() + 1); + if (file == ".hg") return false; + // FIXME: filter out directories with no tracked files. + if (S_ISDIR(st.st_mode)) return true; + return files.count(file); + }; + + hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + + return hgInfo; + } + } + + if (rev == "") rev = "default"; + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. FIXME: don't do this if "rev" is a hash and we + fetched it previously */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + st.st_mtime < now - settings.tarballTtl) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); + + if (pathExists(cacheDir)) { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); + } + + writeFile(stampFile, ""); + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + HgInfo hgInfo; + hgInfo.rev = tokens[0]; + hgInfo.revCount = std::stoull(tokens[1]); + hgInfo.branch = tokens[2]; + + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == hgInfo.rev); + + hgInfo.storePath = json["storePath"]; + + if (store->isValidPath(hgInfo.storePath)) { + printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); + return hgInfo; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + hgInfo.storePath = store->addToStore(name, tmpDir); + + nlohmann::json json; + json["storePath"] = hgInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["branch"] = hgInfo.branch; + json["rev"] = hgInfo.rev; + json["revCount"] = hgInfo.revCount; + + writeFile(storeLink, json.dump()); + + return hgInfo; +} + +static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string url; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); + } + + if (url.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); + + } else + url = state.coerceToString(pos, *args[0], context, false, false); + + if (!isUri(url)) url = absPath(url); + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto hgInfo = exportMercurial(state.store, url, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); + mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + v.attrs->sort(); +} + +static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); + +} diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc index dc526485f..879e3656a 100644 --- a/src/libexpr/primops/fetchgit.cc +++ b/src/libexpr/primops/fetchgit.cc @@ -106,10 +106,9 @@ GitInfo exportGit(ref store, const std::string & uri, std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); + PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken try { - // FIXME: doesn't handle empty lines auto json = nlohmann::json::parse(readFile(storeLink)); assert(json["name"] == name && json["rev"] == gitInfo.rev); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 579a5e8c1..70f9b1f5e 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -707,7 +707,7 @@ bool isUri(const string & s) size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh new file mode 100644 index 000000000..5e4782658 --- /dev/null +++ b/tests/fetchMercurial.sh @@ -0,0 +1,73 @@ +source common.sh + +if [[ -z $(type -p hg) ]]; then + echo "Mercurial not installed; skipping Mercurial tests" + exit 0 +fi + +clearStore + +repo=$TEST_ROOT/hg + +rm -rfv $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg + +hg init $repo +echo '[ui]' >> $repo/.hg/hgrc +echo 'username = Foobar ' >> $repo/.hg/hgrc + +echo utrecht > $repo/hello +hg add --cwd $repo hello +hg commit --cwd $repo -m 'Bla1' +rev1=$(hg log --cwd $repo -r tip --template '{node}') + +echo world > $repo/hello +hg commit --cwd $repo -m 'Bla2' +rev2=$(hg log --cwd $repo -r tip --template '{node}') + +hg log --cwd $repo + +hg log --cwd $repo -r tip --template '{node}\n' + +path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $(cat $path/hello) = world ]] + +# Fetch again. This should be cached. +mv $repo ${repo}-tmp +path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] + +# But with TTL 0, it should fail. +(! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)") + +mv ${repo}-tmp $repo + +# Using a clean working tree should produce the same result. +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[[ $path = $path2 ]] + +# Using an unclean tree should yield the tracked but uncommitted changes. +echo foo > $repo/foo +echo bar > $repo/bar +hg add --cwd $repo foo +hg rm --cwd $repo hello + +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[ ! -e $path2/hello ] +[ ! -e $path2/bar ] +[[ $(cat $path2/foo) = foo ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# ... unless we're using an explicit rev. +path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") +[[ $path = $path3 ]] + +# Committing should not affect the store path. +hg commit --cwd $repo -m 'Bla3' + +path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path2 = $path4 ]] diff --git a/tests/local.mk b/tests/local.mk index 6160b04c2..fced281ee 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -15,7 +15,8 @@ nix_tests = \ linux-sandbox.sh \ build-remote.sh \ nar-index.sh \ - structured-attrs.sh + structured-attrs.sh \ + fetchMercurial.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x))